diff --git a/packages/media/src/__snapshots__/compile.test.ts.snap b/packages/media/src/__snapshots__/compile.test.ts.snap new file mode 100644 index 0000000..94005f5 --- /dev/null +++ b/packages/media/src/__snapshots__/compile.test.ts.snap @@ -0,0 +1,491 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`compileRenderPlan > compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "subtitles_srt", + "path": "/work/render-1/captions.srt", + }, + { + "kind": "subtitles_vtt", + "path": "/work/render-1/captions.vtt", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "final", + "steps": [ + { + "content": "[Script Info] +ScriptType: v4.00+ +PlayResX: 1080 +PlayResY: 1920 +WrapStyle: 2 +ScaledBorderAndShadow: yes + +[V4+ Styles] +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding +Style: Default,Example Sans,96,&H00FFFFFF,&H00FFFFFF,&H00000000,&H80000000,0,0,0,0,100,100,0,0,1,2,0,2,92,168,422,1 + +[Events] +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text +Dialogue: 0,0:00:00.20,0:00:01.80,Default,,0,0,0,,Wait for this! +Dialogue: 0,0:00:02.10,0:00:03.40,Default,,0,0,0,,Here it is --> now. +", + "id": "captions-ass", + "kind": "write_file", + "path": "/work/render-1/captions.ass", + }, + { + "content": "1 +00:00:00,200 --> 00:00:01,800 +Wait for this! + +2 +00:00:02,100 --> 00:00:03,400 +Here it is -> now. +", + "id": "captions-srt", + "kind": "write_file", + "path": "/work/render-1/captions.srt", + }, + { + "content": "WEBVTT + +00:00:00.200 --> 00:00:01.800 +Wait for this! + +00:00:02.100 --> 00:00:03.400 +Here it <b>is</b> --> now. +", + "id": "captions-vtt", + "kind": "write_file", + "path": "/work/render-1/captions.vtt", + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "5", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-i", + "/brand/watermark.png", + "-filter_complex", + "[0:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,170.82\\,if(lt(t\\,1.5)\\,170.82+(167.81)*(t-0)/1.5\\,if(lt(t\\,2)\\,338.62+(19.97)*(t-1.5)/0.5\\,358.59))):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,1.5)\\,0+(0)*(t-0)/1.5\\,if(lt(t\\,2)\\,0+(0)*(t-1.5)/0.5\\,0))),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,1232\\,if(lt(t\\,2)\\,1232+(0)*(t-0)/2\\,1232)):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,2)\\,0+(0)*(t-0)/2\\,0)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c2];[v_c1][a_c1][v_c2][a_c2]concat=n=2:v=1:a=1[cv1][ca2];[2:v]format=rgba,scale=162:-2,colorchannelmixer=aa=0.6[img3];[cv1][img3]overlay=x=main_w-overlay_w-152:y=214:enable=between(t\\,0\\,4)[ov4];[ov4]ass=filename=/work/render-1/captions.ass:fontsdir=/brand/fonts[cap5];[cap5]format=yuv420p[vout];[ca2]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "20", + "-profile:v", + "high", + "-maxrate", + "8000k", + "-bufsize", + "16000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "192k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-metadata", + "title=Example: teaser", + "-t", + "4", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 4000, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "1.333", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; + +exports[`compileRenderPlan > compiles transitions, music ducking, voice-over and fades 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "draft", + "steps": [ + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "2", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "3", + "-t", + "1", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "4", + "-t", + "1", + "-i", + "/media/asset-b/original.mov", + "-ss", + "0", + "-t", + "8", + "-i", + "/brand/music/bed.wav", + "-ss", + "0", + "-t", + "2", + "-i", + "/work/render-1/voiceover.wav", + "-filter_complex", + "[0:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/0.5,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=4,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,atempo=0.5,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=4,asetpts=PTS-STARTPTS[a_c2];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c3];[2:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c3];[3:v]crop=w=608:h=1080:x=656:y=0,scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c4];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c4];[v_c1][v_c2]xfade=transition=fade:duration=0.5:offset=1.5[xv1];[a_c1][a_c2]acrossfade=d=0.5:c1=tri:c2=tri[xa2];color=c=black:s=1080x1920:r=30,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=0.5,setpts=PTS-STARTPTS[gv3];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=0.5,asetpts=PTS-STARTPTS[ga4];[xv1][xa2][gv3][ga4][v_c3][a_c3][v_c4][a_c4]concat=n=4:v=1:a=1[cv5][ca6];[cv5]format=yuv420p[vout];[4:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,volume=-8dB,afade=t=in:st=0:d=0.5,afade=t=out:st=7:d=1[ac_bed];[5:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,adelay=1000|1000[ac_vo];[ca6][ac_vo]amix=inputs=2:duration=first:normalize=0[speech];[speech]asplit=2[speech_mix][speech_key];[ac_bed]anull[duck_bus];[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked];[speech_mix][ducked]amix=inputs=2:duration=first:normalize=0[mixed];[mixed]loudnorm=I=-16:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "28", + "-profile:v", + "high", + "-maxrate", + "3000k", + "-bufsize", + "6000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "128k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-t", + "8", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 8000, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "2.667", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; + +exports[`compileRenderPlan > compiles voice disguise, metadata stripping, intro/outro and text overlays 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "final", + "steps": [ + { + "fps": 30, + "height": 1920, + "id": "motion-1", + "kind": "motion", + "outPath": "/work/render-1/motion-1.mov", + "template": { + "durationMs": 1000, + "params": {}, + "templateId": "outro-simple", + }, + "width": 1080, + }, + { + "fps": 30, + "height": 1920, + "id": "motion-2", + "kind": "motion", + "outPath": "/work/render-1/motion-2.mov", + "template": { + "durationMs": 1000, + "params": {}, + "templateId": "lower-third-a", + }, + "width": 1080, + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "3", + "-i", + "/media/asset-a/proxy.mp4", + "-i", + "/work/render-1/intro.mov", + "-i", + "/work/render-1/motion-1.mov", + "-i", + "/work/render-1/motion-2.mov", + "-filter_complex", + "[0:v]deshake,crop=w=406:h=720:x=if(lt(t\\,0)\\,373\\,if(lt(t\\,3)\\,373+(768)*(t-0)/3\\,1141)):y=if(lt(t\\,0)\\,180\\,if(lt(t\\,3)\\,180+(0)*(t-0)/3\\,180)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,eq=brightness=0.05:contrast=1:saturation=1.1,colorbalance=rm=0.02:bm=-0.02,lut3d=file=/brand/look\\\\\\'s.cube,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=3,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[a_c1];[a_c1]rubberband=pitch=0.793701:formant=preserved,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[vd1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1.5,setpts=PTS-STARTPTS[intro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1.5,asetpts=PTS-STARTPTS[intro_a];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[outro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[outro_a];[intro_v][intro_a][v_c1][vd1][outro_v][outro_a]concat=n=3:v=1:a=1[prog_v][prog_a];[3:v]format=yuva420p,scale=1080:1920,setpts=PTS-STARTPTS+0/TB[mo2];[prog_v][mo2]overlay=x=0:y=0:eof_action=pass:enable=between(t\\,0\\,1)[ov3];color=c=0xFFAA00:s=1080x10:r=30:d=5.5[pb4];[ov3][pb4]overlay=x=-overlay_w+main_w*min(1\\,(t-1.5)/3):y=1504:enable=between(t\\,1.5\\,4.5)[ov5];[ov5]drawtext=fontfile=/brand/fonts/ExampleSans.ttf:text=Ana\\\\\\'s\\\\: \\[studio\\]\\; 100\\\\\\\\% real:fontsize=77:fontcolor=0xFFFFFF:x=(w-text_w)/2:y=h-text_h-560:box=1:boxcolor=0x112233@0.85:boxborderw=31:enable=between(t\\,2\\,3.5)[ov6];[ov6]format=yuv420p[vout];[prog_a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "20", + "-profile:v", + "high", + "-maxrate", + "8000k", + "-bufsize", + "16000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "192k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-t", + "5.5", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 5500, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "2.5", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; diff --git a/packages/media/src/clustering.test.ts b/packages/media/src/clustering.test.ts new file mode 100644 index 0000000..c2e333d --- /dev/null +++ b/packages/media/src/clustering.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from "vitest"; +import type { ClusterAsset } from "./clustering.js"; +import { + buildSessions, + calendarDay, + clusterFaces, + clusterThemes, + ruleLabel, +} from "./clustering.js"; + +const at = (iso: string) => new Date(iso); + +function asset( + id: string, + captured: string | null, + extra: Partial = {}, +): ClusterAsset { + return { + id, + kind: "photo", + capturedAt: captured ? at(captured) : null, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + embedding: null, + ...extra, + }; +} + +const vec = (...values: number[]) => new Float32Array(values); + +describe("buildSessions", () => { + it("splits on gaps longer than 45 minutes and sorts by capture time", () => { + const sessions = buildSessions([ + asset("c", "2026-05-01T10:40:00Z"), + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:20:00Z"), + asset("d", "2026-05-01T11:26:00Z"), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]); + expect(sessions[0]!.key).toBe("session:a"); + expect(sessions[0]!.startAt).toEqual(at("2026-05-01T10:00:00Z")); + expect(sessions[0]!.endAt).toEqual(at("2026-05-01T10:40:00Z")); + }); + + it("keeps a gap of exactly 45 minutes in one session", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:45:00Z"), + ]); + expect(sessions).toHaveLength(1); + }); + + it("puts undated assets into one trailing session", () => { + const sessions = buildSessions([ + asset("z", null), + asset("a", "2026-05-01T10:00:00Z"), + asset("m", null), + asset("bad", null, { capturedAt: new Date(Number.NaN) }), + ]); + expect(sessions.map((s) => s.key)).toEqual(["session:a", "session:undated"]); + expect(sessions[1]!.assetIds).toEqual(["bad", "m", "z"]); + expect(sessions[1]!.startAt).toBeNull(); + }); + + it("ignores duplicate ids and mixes photos and videos", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z"), + asset("v", "2026-05-01T10:05:00Z", { kind: "video" }), + asset("a", "2026-05-01T23:00:00Z"), + asset("l", "2026-05-01T10:06:00Z", { kind: "live_photo" }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "v", "l"]]); + }); + + it("splits on a location jump over 2 km and reports a rounded centroid", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z", { gpsLat: 10.0, gpsLon: 20.0 }), + asset("b", "2026-05-01T10:05:00Z"), + asset("c", "2026-05-01T10:10:00Z", { gpsLat: 10.004, gpsLon: 20.004 }), + asset("d", "2026-05-01T10:15:00Z", { gpsLat: 10.1, gpsLon: 20.0 }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]); + expect(sessions[0]!.centroid).toEqual({ lat: 10, lon: 20 }); + expect(sessions[1]!.centroid).toEqual({ lat: 10.1, lon: 20 }); + }); + + it("breaks on a calendar-day change only with a gap over 3 hours, in the given time zone", () => { + const assets = [asset("a", "2026-05-01T22:30:00Z"), asset("b", "2026-05-02T02:00:00Z")]; + // Long gap rule disabled so only the day-break rule decides. + const options = { gapMinutes: 24 * 60 }; + expect(buildSessions(assets, options)).toHaveLength(2); + // In UTC-10 both are on 1 May: one session. + expect(buildSessions(assets, { ...options, timeZone: "Pacific/Honolulu" })).toHaveLength(1); + // Crossing midnight with a short gap stays one session. + expect( + buildSessions( + [asset("x", "2026-05-01T23:50:00Z"), asset("y", "2026-05-02T00:20:00Z")], + options, + ), + ).toHaveLength(1); + expect(calendarDay(at("2026-05-01T22:30:00Z"), "Europe/Berlin")).toBe("2026-05-02"); + }); + + it("merges neighbours with similar embeddings and a gap under 2 hours", () => { + const red = vec(1, 0, 0); + const blue = vec(0, 1, 0); + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z", { embedding: red }), + asset("b", "2026-05-01T11:00:00Z", { embedding: vec(0.95, 0.05, 0) }), + asset("c", "2026-05-01T12:00:00Z", { embedding: blue }), + asset("d", "2026-05-01T15:00:00Z", { embedding: blue }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b"], ["c"], ["d"]]); + }); + + it("is deterministic for any input order", () => { + const assets = [ + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:00:00Z"), + asset("c", "2026-05-02T10:00:00Z"), + asset("d", null), + ]; + const forward = buildSessions(assets); + const reversed = buildSessions([...assets].reverse()); + expect(reversed).toEqual(forward); + expect(forward[0]!.assetIds).toEqual(["a", "b"]); + }); +}); + +describe("clusterThemes and clusterFaces", () => { + const items = [ + { id: "p3", vector: vec(0.9, 0.1, 0) }, + { id: "p1", vector: vec(1, 0, 0) }, + { id: "q1", vector: vec(0, 1, 0) }, + { id: "p2", vector: vec(0.95, 0.05, 0) }, + { id: "q2", vector: vec(0.05, 0.95, 0) }, + { id: "r1", vector: vec(0, 0, 1) }, + ]; + + it("groups by average linkage with stable keys, independent of input order", () => { + const themes = clusterThemes(items); + expect(themes).toEqual([ + { key: "theme:p1", ids: ["p1", "p2", "p3"] }, + { key: "theme:q1", ids: ["q1", "q2"] }, + { key: "theme:r1", ids: ["r1"] }, + ]); + expect(clusterThemes([...items].reverse())).toEqual(themes); + expect(clusterThemes(items, { minSize: 2 }).map((t) => t.key)).toEqual([ + "theme:p1", + "theme:q1", + ]); + expect(clusterThemes(items, { similarity: -1 })).toHaveLength(1); + expect(clusterThemes([])).toEqual([]); + }); + + it("uses average rather than single linkage", () => { + // b is close to a and c, but a and c are far apart: average linkage keeps c out. + const chain = [ + { id: "a", vector: vec(1, 0) }, + { id: "b", vector: vec(Math.cos(0.4), Math.sin(0.4)) }, + { id: "c", vector: vec(Math.cos(1.0), Math.sin(1.0)) }, + ]; + expect(clusterThemes(chain, { similarity: 0.8 }).map((g) => g.ids)).toEqual([ + ["a", "b"], + ["c"], + ]); + }); + + it("returns face groups with a mean centroid", () => { + const faces = clusterFaces([ + { id: "t1", vector: vec(1, 0) }, + { id: "t2", vector: vec(0.8, 0.2) }, + { id: "t3", vector: vec(0, 1) }, + ]); + expect(faces.map((f) => f.key)).toEqual(["person:t1", "person:t3"]); + expect(Array.from(faces[0]!.centroid)).toEqual([0.8999999761581421, 0.10000000149011612]); + expect(() => + clusterFaces([ + { id: "x", vector: vec(1) }, + { id: "y", vector: vec(1, 2) }, + ]), + ).toThrow(); + }); +}); + +describe("ruleLabel", () => { + it("combines the dominant album with a locale-aware date range", () => { + const label = ruleLabel( + { + startAt: at("2026-05-01T10:00:00Z"), + endAt: at("2026-05-03T10:00:00Z"), + albumNames: ["Trip", "Favorites", "Trip"], + }, + "en-US", + ); + expect(label.startsWith("Trip · ")).toBe(true); + expect(label).toMatch(/May 1\D+3, 2026/); + expect( + ruleLabel({ startAt: at("2026-05-01T10:00:00Z"), endAt: null, albumNames: [] }, "de-DE"), + ).toBe("01.05.2026"); + expect(ruleLabel({ startAt: null, endAt: null, albumNames: [" "] })).toBe(""); + expect(ruleLabel({ startAt: null, endAt: null, albumNames: ["B", "A"] })).toBe("A"); + }); +}); diff --git a/packages/media/src/compile.ffmpeg.test.ts b/packages/media/src/compile.ffmpeg.test.ts new file mode 100644 index 0000000..cd6051e --- /dev/null +++ b/packages/media/src/compile.ffmpeg.test.ts @@ -0,0 +1,322 @@ +import { execFile, execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import type { RenderPlan } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { CompileInput } from "./compile.js"; +import { compileRenderPlan, escapeFilterText, escapeFilterValue } from "./compile.js"; +import { edlDurationMs } from "./edl.js"; +import { findMetadataLeaks } from "./gate.js"; +import { testEdl, testKit, testProfile } from "./test-fixtures.js"; + +const run = promisify(execFile); + +function onPath(binary: string): boolean { + try { + execFileSync(binary, ["-version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +const hasFfmpeg = onPath("ffmpeg") && onPath("ffprobe"); +const hasRubberband = + hasFfmpeg && + execFileSync("ffmpeg", ["-hide_banner", "-filters"]).toString().includes(" rubberband "); + +async function ffmpeg(args: string[]): Promise { + await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]); +} + +async function execute(plan: RenderPlan): Promise { + for (const step of plan.steps) { + if (step.kind === "write_file") await writeFile(step.path, step.content); + else if (step.kind === "ffmpeg") await run("ffmpeg", ["-loglevel", "error", ...step.argv]); + else throw new Error(`unexpected ${step.kind} step`); + } +} + +interface Probe { + durationMs: number; + width: number; + height: number; + tags: Record; +} + +async function probe(file: string): Promise { + const { stdout } = await run("ffprobe", [ + "-v", + "error", + "-show_format", + "-show_streams", + "-of", + "json", + file, + ]); + const json = JSON.parse(stdout) as { + format: { duration: string; tags?: Record }; + streams: Array<{ + codec_type: string; + width?: number; + height?: number; + tags?: Record; + }>; + }; + const tags: Record = {}; + for (const [key, value] of Object.entries(json.format.tags ?? {})) tags[`format.${key}`] = value; + json.streams.forEach((stream, index) => { + for (const [key, value] of Object.entries(stream.tags ?? {})) + tags[`stream.${index}.${key}`] = value; + }); + const video = json.streams.find((s) => s.codec_type === "video"); + return { + durationMs: Number(json.format.duration) * 1000, + width: video?.width ?? 0, + height: video?.height ?? 0, + tags, + }; +} + +describe.skipIf(!hasFfmpeg)("compiled plans run on ffmpeg", () => { + let dir: string; + let source: string; + let watermark: string; + let music: string; + let intro: string; + let lut: string; + + beforeAll(async () => { + dir = await mkdtemp(path.join(tmpdir(), "media-compile-")); + source = path.join(dir, "source.mp4"); + watermark = path.join(dir, "watermark.png"); + music = path.join(dir, "music.wav"); + intro = path.join(dir, "intro.mov"); + lut = path.join(dir, "brand look.cube"); + // Identity 2x2x2 LUT. + const lattice = []; + for (const b of [0, 1]) + for (const g of [0, 1]) for (const r of [0, 1]) lattice.push(`${r} ${g} ${b}`); + await writeFile(lut, `LUT_3D_SIZE 2\n${lattice.join("\n")}\n`); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc2=size=480x270:rate=30:duration=8", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=8", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + "-shortest", + // Tags a phone would write; the export must drop every one of them. + "-metadata", + "creation_time=2020-01-01T10:00:00Z", + "-metadata", + "location=+00.0000+000.0000/", + "-metadata", + "make=ExampleCam", + source, + ]); + await ffmpeg(["-f", "lavfi", "-i", "color=c=white:s=64x32", "-frames:v", "1", watermark]); + await ffmpeg(["-f", "lavfi", "-i", "sine=frequency=220:duration=6", music]); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc=size=320x180:rate=30:duration=1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + intro, + ]); + }); + + afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }); + }); + + const baseInput = (overrides: Partial): CompileInput => ({ + edl: testEdl(), + profile: testProfile(), + brandKit: testKit(), + quality: "draft", + workDir: dir, + sources: new Map([ + [ + "asset-a", + { + path: source, + width: 480, + height: 270, + durationMs: 8000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + ]), + transcripts: new Map([ + [ + "asset-a", + [ + { startMs: 200, endMs: 600, text: "Wait" }, + { startMs: 650, endMs: 1200, text: "for" }, + { startMs: 1250, endMs: 1800, text: "this!" }, + { startMs: 5100, endMs: 5600, text: "Here" }, + { startMs: 5650, endMs: 6400, text: "it is." }, + ], + ], + ]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.3, cy: 0.5, strength: 1 }, + { tMs: 6000, cx: 0.7, cy: 0.5, strength: 1 }, + ], + ], + ]), + motionClips: new Map(), + files: { + lutPath: null, + watermarkPath: watermark, + fonts: new Map(), + audio: new Map([["music-1", music]]), + }, + videoEncoder: "libx264", + ...overrides, + }); + + it("renders 16:9 to 9:16 with auto-reframe, captions, watermark, ducked music and clean metadata", async () => { + const edl = testEdl({ + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 1500, reframe: "auto" }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: "music-1", + inMs: 0, + outMs: 3500, + atMs: 0, + gainDb: -6, + fadeOutMs: 500, + duckUnderSpeech: true, + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }], + captions: { mode: "both" }, + overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 3500 }], + metadata: { title: "Example teaser" }, + }); + const plan = compileRenderPlan(baseInput({ edl })); + await execute(plan); + const video = plan.outputs.find((o) => o.kind === "video")!.path; + const result = await probe(video); + expect(result.width).toBe(1080); + expect(result.height).toBe(1920); + expect(Math.abs(result.durationMs - edlDurationMs(edl))).toBeLessThanOrEqual(1000 / 30 + 1); + const keys = Object.keys(result.tags).map((k) => k.toLowerCase()); + expect(keys.some((k) => k.includes("creation_time") || k.includes("location"))).toBe(false); + expect(findMetadataLeaks(result.tags)).toEqual([]); + expect(result.tags["format.title"]).toBe("Example teaser"); + + const srt = await readFile(plan.outputs.find((o) => o.kind === "subtitles_srt")!.path, "utf8"); + expect(srt).toContain("Wait for this!"); + const poster = plan.outputs.find((o) => o.kind === "poster")!.path; + expect(existsSync(poster)).toBe(true); + expect((await probe(poster)).tags).toEqual({}); + }); + + it("renders an intro, voice disguise and an escaped text overlay", async () => { + const edl = testEdl({ + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 1000, + outMs: 3000, + atMs: 0, + reframe: "none", + stabilize: true, + color: { exposure: 0.3, temperature: -0.4, saturation: 1.2, useBrandLut: true }, + }, + ], + }, + intro: { templateId: "intro-minimal", durationMs: 1000 }, + overlays: [ + { + id: "t1", + kind: "text", + atMs: 1000, + durationMs: 1500, + text: "50% off: it's [0:v]; rm -rf / %{pts} \\ done", + box: { x: 0.1, y: 0.3, w: 0.8, h: 0.05 }, + }, + { id: "lt", kind: "lower_third", atMs: 1200, durationMs: 1000, text: "Lower third" }, + { id: "pb", kind: "progress_bar", atMs: 1000, durationMs: 2000 }, + ], + redactions: { voice: { method: hasRubberband ? "pitch" : "formant", semitones: 3 } }, + metadata: {}, + }); + const plan = compileRenderPlan( + baseInput({ + edl, + motionClips: new Map([["intro", intro]]), + quality: "final", + files: { ...baseInput({}).files, lutPath: lut }, + }), + ); + await execute(plan); + const result = await probe(plan.outputs[0]!.path); + expect(result.width).toBe(1080); + expect(Math.abs(result.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1); + expect(findMetadataLeaks(result.tags)).toEqual([]); + expect(Object.keys(result.tags).some((k) => k.endsWith("title"))).toBe(false); + }); + + it("draws escaped text exactly like the literal text", async () => { + const text = "it's 50% [0:v]; a,b=c \\ %{pts} 'q'"; + const literal = path.join(dir, "literal.txt"); + await writeFile(literal, text); + const frame = async (filter: string) => { + const { stdout } = await run("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `color=c=black:s=320x80:d=0.1,${filter}`, + "-frames:v", + "1", + "-f", + "framemd5", + "-", + ]); + return stdout.split("\n").filter((line) => line && !line.startsWith("#")); + }; + const escaped = await frame(`drawtext=text=${escapeFilterText(text)}:fontcolor=white:x=4:y=20`); + const reference = await frame( + `drawtext=expansion=none:textfile=${escapeFilterValue(literal)}:fontcolor=white:x=4:y=20`, + ); + const blank = await frame("null"); + expect(escaped).toEqual(reference); + expect(escaped).not.toEqual(blank); + }); +}); diff --git a/packages/media/src/compile.test.ts b/packages/media/src/compile.test.ts new file mode 100644 index 0000000..1a14ebf --- /dev/null +++ b/packages/media/src/compile.test.ts @@ -0,0 +1,576 @@ +import type { RenderPlan } from "@rakazo/contracts"; +import { RenderPlanSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { CaptionWord, CompileInput } from "./compile.js"; +import { + atempoChain, + autoReframe, + buildCaptionCues, + compileRenderPlan, + escapeFilterText, + escapeFilterValue, + outputCaptionWords, + reframeExcluding, + toAss, + toSrt, + toVtt, +} from "./compile.js"; +import { testEdl, testKit, testProfile } from "./test-fixtures.js"; + +const words: CaptionWord[] = [ + { startMs: 200, endMs: 600, text: "Wait" }, + { startMs: 650, endMs: 1200, text: "for" }, + { startMs: 1250, endMs: 1800, text: "this!" }, + { startMs: 5100, endMs: 5600, text: "Here" }, + { startMs: 5650, endMs: 6400, text: "it is --> now." }, +]; + +function input(overrides: Partial = {}): CompileInput { + return { + edl: testEdl(), + profile: testProfile(), + brandKit: testKit(), + quality: "final", + workDir: "/work/render-1", + sources: new Map([ + [ + "asset-a", + { + path: "/media/asset-a/proxy.mp4", + width: 1920, + height: 1080, + durationMs: 10_000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + [ + "asset-b", + { + path: "/media/asset-b/original.mov", + width: 1080, + height: 1920, + durationMs: 8000, + fps: 60, + hasAudio: false, + rotation: 90, + }, + ], + ]), + transcripts: new Map([["asset-a", words]]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.2, cy: 0.5, strength: 1 }, + { tMs: 1500, cx: 0.4, cy: 0.5, strength: 1 }, + { tMs: 5500, cx: 0.8, cy: 0.4, strength: 0.5 }, + ], + ], + ]), + motionClips: new Map(), + files: { + lutPath: "/brand/look's.cube", + watermarkPath: "/brand/watermark.png", + fonts: new Map([["font-1", "/brand/fonts/ExampleSans.ttf"]]), + audio: new Map([ + ["music-1", "/brand/music/bed.wav"], + ["vo-1", "/work/render-1/voiceover.wav"], + ]), + }, + videoEncoder: "libx264", + ...overrides, + }; +} + +function ffmpegStep(plan: RenderPlan, id = "render") { + const step = plan.steps.find((s) => s.id === id); + if (step?.kind !== "ffmpeg") throw new Error(`no ffmpeg step ${id}`); + return step; +} + +function graphOf(plan: RenderPlan): string { + const argv = ffmpegStep(plan).argv; + return argv[argv.indexOf("-filter_complex") + 1]!; +} + +describe("compileRenderPlan", () => { + it("compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + captions: { mode: "both", presetId: "clean" }, + overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 }], + metadata: { title: "Example: teaser" }, + }), + }), + ); + expect(RenderPlanSchema.parse(plan)).toEqual(plan); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("crop=w=608:h=1080:x="); + expect(graph).toContain("scale=1080:1920"); + expect(graph).toContain("loudnorm=I=-14:TP=-1:LRA=11:linear=true"); + expect(graph).toContain("colorchannelmixer=aa=0.6"); + expect(graph).toContain("ass=filename=/work/render-1/captions.ass"); + expect(plan.outputs.map((o) => o.kind)).toEqual([ + "video", + "subtitles_srt", + "subtitles_vtt", + "poster", + ]); + const argv = ffmpegStep(plan).argv; + for (const flag of [ + "-map_metadata", + "-map_chapters", + "+bitexact", + "+faststart", + "yuv420p", + "bt709", + ]) { + expect(argv).toContain(flag); + } + expect(argv).toContain("title=Example: teaser"); + expect(argv.join(" ")).toContain("-c:a aac -b:a 192k -ar 48000"); + }); + + it("compiles transitions, music ducking, voice-over and fades", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" }, + { + id: "c2", + assetId: "asset-a", + inMs: 2000, + outMs: 4000, + atMs: 1500, + reframe: "none", + speed: 0.5, + }, + { + id: "c3", + assetId: "asset-a", + inMs: 3000, + outMs: 4000, + atMs: 6000, + reframe: "none", + }, + { + id: "c4", + assetId: "asset-b", + inMs: 4000, + outMs: 5000, + atMs: 7000, + reframe: "auto", + }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: "music-1", + inMs: 0, + outMs: 8000, + atMs: 0, + gainDb: -8, + fadeInMs: 500, + fadeOutMs: 1000, + duckUnderSpeech: true, + }, + { id: "vo", kind: "voiceover", artifactId: "vo-1", inMs: 0, outMs: 2000, atMs: 1000 }, + ], + }, + transitions: [ + { fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }, + { fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }, + { fromClipId: "c3", toClipId: "c4", kind: "whip", durationMs: 0 }, + ], + targetLufs: -16, + }), + quality: "draft", + }), + ); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("xfade=transition=fade:duration=0.5:offset=1.5"); + expect(graph).toContain("acrossfade=d=0.5"); + expect(graph).toContain("atempo=0.5"); + expect(graph).toContain("color=c=black:s=1080x1920"); // gap from 5.5 s to 6 s + expect(graph).toContain("sidechaincompress"); + expect(graph).toContain("adelay=1000|1000"); + expect(graph).toContain("loudnorm=I=-16"); + expect(ffmpegStep(plan).argv).toEqual(expect.arrayContaining(["veryfast", "28", "3000k"])); + }); + + it("compiles voice disguise, metadata stripping, intro/outro and text overlays", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1500 }, + outro: { templateId: "outro-simple", durationMs: 1000 }, + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 0, + outMs: 3000, + atMs: 0, + reframe: [ + { tMs: 0, cx: 0.3, cy: 0.5, scale: 1 }, + { tMs: 3000, cx: 0.7, cy: 0.5, scale: 1.5 }, + ], + stabilize: true, + color: { exposure: 0.5, temperature: 0.2, saturation: 1.1, useBrandLut: true }, + }, + ], + }, + overlays: [ + { + id: "lt", + kind: "lower_third", + atMs: 2000, + durationMs: 1500, + text: "Ana's: [studio]; 100% real", + }, + { + id: "logo-anim", + kind: "motion", + atMs: 0, + durationMs: 1000, + motion: { templateId: "lower-third-a", durationMs: 1000 }, + }, + { id: "pb", kind: "progress_bar", atMs: 1500, durationMs: 3000 }, + ], + redactions: { voice: { method: "pitch", semitones: -4 } }, + metadata: {}, + }), + motionClips: new Map([["intro", "/work/render-1/intro.mov"]]), + }), + ); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("rubberband=pitch=0.793701:formant=preserved"); + expect(graph).toContain("deshake"); + expect(graph).toContain("lut3d=file=/brand/look\\\\\\'s.cube"); + expect(graph).toContain("concat=n=3:v=1:a=1[prog_v][prog_a]"); + expect(plan.steps.filter((s) => s.kind === "motion").map((s) => s.id)).toEqual([ + "motion-1", + "motion-2", + ]); + const argv = ffmpegStep(plan).argv; + expect(argv.some((a) => a.startsWith("title="))).toBe(false); + expect(argv).toEqual(expect.arrayContaining(["-map_metadata", "-1", "-map_chapters", "-1"])); + }); + + it("uses the asetrate chain for formant disguise and HEVC settings for HEVC profiles", () => { + const plan = compileRenderPlan( + input({ + profile: testProfile({ + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "hevc", + maxBitrateKbps: 6000, + maxFileSizeMb: 100, + }, + export: { poster: false }, + }), + edl: testEdl({ redactions: { voice: { method: "formant", semitones: 12 } }, metadata: {} }), + videoEncoder: "hevc_videotoolbox", + }), + ); + expect(graphOf(plan)).toContain("asetrate=96000,aresample=48000,atempo=0.5"); + const argv = ffmpegStep(plan).argv; + expect(argv).toEqual(expect.arrayContaining(["-tag:v", "hvc1", "-b:v", "4800k"])); + expect(argv).toContain("filter_units=remove_types=39|40"); + expect(plan.outputs.map((o) => o.kind)).toEqual(["video"]); + }); + + it("only ever uses SafeId labels and keeps text out of argv", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + overlays: [ + { id: "t1", kind: "text", atMs: 0, durationMs: 1000, text: "'; rm -rf / [0:v] %{pts}" }, + ], + }), + }), + ); + const graph = graphOf(plan); + for (const label of graph.matchAll(/\[([^\]\\]*)\]/g)) { + expect(label[1]).toMatch(/^[a-z0-9_:-]+$/); + } + expect(graph).toContain("text=\\\\\\'\\; rm -rf / \\[0\\\\:v\\] \\\\\\\\%{pts}"); + expect(ffmpegStep(plan).argv.every((arg) => !arg.includes("\0"))).toBe(true); + }); + + it("refuses unsafe or impossible input instead of guessing", () => { + expect(() => + compileRenderPlan( + input({ + edl: { + ...testEdl(), + overlays: [{ id: "BAD;ID", kind: "text", atMs: 0, durationMs: 1, text: "x" }], + } as never, + }), + ), + ).toThrow(); + expect(() => compileRenderPlan(input({ videoEncoder: "libx265" }))).toThrow( + /does not produce h264/, + ); + expect(() => compileRenderPlan(input({ sources: new Map() }))).toThrow(/no source/); + expect(() => + compileRenderPlan( + input({ + edl: testEdl({ overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 1000 }] }), + files: { ...input().files, watermarkPath: null }, + }), + ), + ).toThrow(/watermark/); + }); + + it("moves very large filtergraphs into a script file", () => { + const video = Array.from({ length: 300 }, (_, i) => ({ + id: `c${i}`, + assetId: "asset-a", + inMs: 0, + outMs: 100, + atMs: i * 100, + reframe: Array.from({ length: 8 }, (_, k) => ({ + tMs: k * 12, + cx: 0.3 + k * 0.05, + cy: 0.5, + scale: 1, + })), + })); + const plan = compileRenderPlan(input({ edl: testEdl({ tracks: { video } }) })); + const script = plan.steps.find((s) => s.id === "filtergraph"); + expect(script?.kind).toBe("write_file"); + expect(ffmpegStep(plan).argv).toContain("-filter_complex_script"); + }); +}); + +describe("escaping", () => { + it("escapes every filtergraph and drawtext special character", () => { + expect(escapeFilterText("plain text")).toBe("plain text"); + expect(escapeFilterText("a:b")).toBe("a\\\\:b"); + expect(escapeFilterText("a,b;c")).toBe("a\\,b\\;c"); + expect(escapeFilterText("[0:v]")).toBe("\\[0\\\\:v\\]"); + expect(escapeFilterText("x=y")).toBe("x\\\\=y"); + expect(escapeFilterText("it's")).toBe("it\\\\\\'s"); + expect(escapeFilterText("100%")).toBe("100\\\\\\\\%"); + expect(escapeFilterText("a\\b")).toBe("a\\\\\\\\\\\\\\\\b"); + expect(escapeFilterText(" line one\r\nline two\u0007 ")).toBe("line one\nline two"); + }); + + /** av_get_token: one parser level; returns the token and what it left unconsumed. */ + function getToken(value: string, terms: string): { token: string; rest: string } { + let token = ""; + let i = 0; + while (i < value.length && !terms.includes(value[i]!)) { + const char = value[i++]!; + if (char === "\\" && i < value.length) token += value[i++]; + else if (char === "'") { + while (i < value.length && value[i] !== "'") token += value[i++]; + i++; + } else token += char; + } + return { token, rest: value.slice(i) }; + } + + function drawtextExpand(value: string): string { + let out = ""; + for (let i = 0; i < value.length; i++) { + if (value[i] === "\\" && i + 1 < value.length) out += value[++i]; + else if (value[i] === "%") throw new Error("expansion sequence reached drawtext"); + else out += value[i]; + } + return out; + } + + it("neutralizes injection attempts", () => { + const attempts = [ + "'; rm -rf /", + "[0:v]", + "%{pts}", + "%{eif:1:d}", + "x:fontfile=/etc/passwd", + "a',drawtext=text='b", + "a\\", + "50% off; [in]null[out]", + ]; + for (const attempt of attempts) { + const graphLevel = getToken(escapeFilterText(attempt), "[],;"); + expect(graphLevel.rest).toBe(""); + const optionLevel = getToken(graphLevel.token, ":"); + expect(optionLevel.rest).toBe(""); + expect(drawtextExpand(optionLevel.token)).toBe(attempt); + } + expect(escapeFilterText("%{pts}")).toBe("\\\\\\\\%{pts}"); + expect(escapeFilterValue("C:\\media\\a,b.cube")).toBe("C\\\\:\\\\\\\\media\\\\\\\\a\\,b.cube"); + expect(() => escapeFilterValue("a\0b")).toThrow(); + }); + + it("chains atempo within its clean range", () => { + expect(atempoChain(1)).toEqual([]); + expect(atempoChain(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]); + expect(atempoChain(3)).toEqual(["atempo=2", "atempo=1.5"]); + expect(atempoChain(4)).toEqual(["atempo=2", "atempo=2"]); + }); +}); + +describe("captions", () => { + it("builds cues by line length, line count, pauses and sentence ends", () => { + const cues = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 1 }); + expect(cues.map((c) => c.text)).toEqual(["Wait for", "this!", "Here", "it is --> now."]); + const two = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 2 }); + expect(two[0]).toMatchObject({ startMs: 200, endMs: 1800, text: "Wait for\nthis!" }); + expect(two[0]!.words).toHaveLength(3); + expect( + buildCaptionCues([{ startMs: 0, endMs: 0, text: " " }], { maxCharsPerLine: 10, maxLines: 2 }), + ).toEqual([]); + const short = buildCaptionCues([{ startMs: 1000, endMs: 1100, text: "Hi" }], { + maxCharsPerLine: 10, + maxLines: 2, + }); + expect(short[0]!.endMs).toBe(1500); + }); + + it("formats SRT and VTT", () => { + const cues = buildCaptionCues(words, { maxCharsPerLine: 32, maxLines: 2 }); + expect(toSrt(cues)).toBe( + "1\n00:00:00,200 --> 00:00:01,800\nWait for this!\n\n2\n00:00:05,100 --> 00:00:06,400\nHere it is -> now.\n", + ); + expect(toVtt(cues)).toBe( + "WEBVTT\n\n00:00:00.200 --> 00:00:01.800\nWait for this!\n\n00:00:05.100 --> 00:00:06.400\nHere it <b>is</b> --> now.\n", + ); + expect(toSrt([{ startMs: 3_723_004, endMs: 3_723_500, text: "a\n\nb", words: [] }])).toBe( + "1\n01:02:03,004 --> 01:02:03,500\na\nb\n", + ); + }); + + it("maps source words to output time across speed and intro", () => { + const edl = testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1000 }, + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 1000, + outMs: 2000, + atMs: 0, + reframe: "none", + speed: 2, + }, + { + id: "c2", + assetId: "asset-a", + inMs: 5000, + outMs: 6000, + atMs: 500, + reframe: "none", + muted: true, + }, + ], + }, + }); + expect(outputCaptionWords(edl, new Map([["asset-a", words]]))).toEqual([ + { startMs: 1125, endMs: 1400, text: "this!" }, + ]); + }); + + it("writes ASS without override injection and with karaoke timing", () => { + const cues = buildCaptionCues( + [ + { startMs: 0, endMs: 400, text: "{\\pos(0,0)}hi" }, + { startMs: 600, endMs: 900, text: "you\\N" }, + ], + { maxCharsPerLine: 32, maxLines: 2 }, + ); + const ass = toAss(cues, { + width: 1080, + height: 1920, + profile: testProfile(), + preset: { + ...testKit().captionPresets[0]!, + template: "karaoke", + highlightColor: "#FFAA00", + uppercase: true, + }, + fallbackTextColor: "#FFFFFF", + position: "lower", + maxCharsPerLine: 32, + wordHighlight: true, + }); + const dialogue = ass.split("\n").find((line) => line.startsWith("Dialogue:"))!; + expect(dialogue).toBe( + "Dialogue: 0,0:00:00.00,0:00:00.90,Default,,0,0,0,,{\\k40}\uFF5B\\\u2060POS(0,0)\uFF5DHI {\\k20}{\\k30}YOU\\\u2060N", + ); + expect(ass).toContain("Style: Default,Example Sans,96,&H0000AAFF,&H00FFFFFF"); + expect(ass).toContain(",2,92,168,422,1"); + }); +}); + +describe("reframing", () => { + const geometry = { + sourceWidth: 1920, + sourceHeight: 1080, + outWidth: 1080, + outHeight: 1920, + durationMs: 4000, + }; + + it("follows the subject smoothly and keeps the crop inside the frame", () => { + const path = autoReframe( + [ + { tMs: 0, cx: 0, cy: 0.5, strength: 1 }, + { tMs: 2000, cx: 0.5, cy: 0.5, strength: 1 }, + { tMs: 4000, cx: 1, cy: 0.5, strength: 1 }, + ], + geometry, + ); + const half = 608 / 1920 / 2; + for (const k of path) { + expect(k.cx).toBeGreaterThanOrEqual(half - 1e-4); + expect(k.cx).toBeLessThanOrEqual(1 - half + 1e-4); + expect(k.cy).toBe(0.5); + expect(k.scale).toBe(1); + } + expect(path[0]!.tMs).toBe(0); + expect(path.at(-1)!.tMs).toBe(4000); + for (let i = 1; i < path.length; i++) + expect(path[i]!.cx).toBeGreaterThanOrEqual(path[i - 1]!.cx); + expect(path.length).toBeLessThanOrEqual(64); + expect(autoReframe([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]); + }); + + it("keeps faces out of the frame or reports that it cannot", () => { + const left = reframeExcluding( + [{ tMs: 100, box: { x: 0.05, y: 0.3, w: 0.15, h: 0.3 } }], + geometry, + ); + expect(left).toHaveLength(1); + const k = left![0]!; + expect(k.tMs).toBe(100); + expect(k.scale).toBe(1); + expect(k.cx - 608 / 1920 / 2).toBeGreaterThanOrEqual(0.22 - 1e-6); + // Faces all over the frame: no crop can exclude them. + const everywhere = [0, 0.25, 0.5, 0.75].map((x) => ({ + tMs: 0, + box: { x, y: 0, w: 0.25, h: 1 }, + })); + expect(reframeExcluding(everywhere, geometry)).toBeNull(); + // A face across the middle forces a tighter crop. + const wide = reframeExcluding([{ tMs: 0, box: { x: 0.2, y: 0.4, w: 0.6, h: 0.2 } }], geometry); + expect(wide![0]!.scale).toBeGreaterThan(1); + expect(reframeExcluding([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]); + }); +}); diff --git a/packages/media/src/edl.test.ts b/packages/media/src/edl.test.ts new file mode 100644 index 0000000..f72e9f9 --- /dev/null +++ b/packages/media/src/edl.test.ts @@ -0,0 +1,337 @@ +import type { EdlError } from "@rakazo/contracts"; +import { EdlErrorSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { EdlContext } from "./edl.js"; +import { edlDurationMs, locateOutputTime, locateOutputTimeAll, validateEdl } from "./edl.js"; +import { testContext, testEdl, testProfile } from "./test-fixtures.js"; + +/** Raw plan (pre-defaults) so cases can break it in any way. */ +function raw(patch: (plan: Record) => void): unknown { + const plan = JSON.parse(JSON.stringify(testEdl())) as Record; + patch(plan); + return plan; +} + +function codes(input: unknown, context: EdlContext = testContext()): EdlError["code"][] { + const result = validateEdl(input, context); + return result.ok ? [] : result.errors.map((e) => e.code); +} + +describe("validateEdl", () => { + it("accepts a valid plan and returns the parsed EDL", () => { + const result = validateEdl(testEdl(), testContext()); + expect(result.ok).toBe(true); + if (result.ok) expect(result.edl.tracks.video).toHaveLength(2); + }); + + const cases: Array<{ + code: EdlError["code"]; + input: unknown; + context?: Partial; + path?: string; + }> = [ + { code: "schema", input: raw((p) => (p.edlVersion = 2)), path: "edlVersion" }, + { + code: "schema", + input: raw((p) => (p.tracks.video[1].id = "c1")), + path: "tracks.video[1].id", + }, + { + code: "schema", + input: raw((p) => (p.tracks.video[0].id = "Bad Id;[x]")), + path: "tracks.video[0].id", + }, + { + code: "unknown_asset", + input: raw((p) => (p.tracks.video[0].assetId = "missing")), + path: "tracks.video[0].assetId", + }, + { + code: "unknown_clip", + input: raw((p) => { + p.transitions = [{ fromClipId: "c1", toClipId: "c9", kind: "cut", durationMs: 0 }]; + }), + path: "transitions[0].toClipId", + }, + { + code: "unknown_clip", + input: raw((p) => { + p.redactions.regions = [ + { + id: "r1", + clipId: "zz", + keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 0.1, h: 0.1 } }], + treatment: "blur", + }, + ]; + }), + }, + { + code: "unknown_artifact", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "sfx", artifactId: "nope", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "unknown_artifact", + input: raw((p) => (p.overlays = [{ id: "w", kind: "watermark", atMs: 0, durationMs: 1000 }])), + context: { brandKit: null }, + }, + { + code: "unknown_mark", + input: raw((p) => (p.redactions.marks = [{ markId: "mark-x", treatment: "blur" }])), + }, + { + code: "unknown_person", + input: raw((p) => { + p.redactions.faces = [{ id: "f", target: { personId: "person-x" }, mode: "blur" }]; + }), + }, + { + code: "unknown_track", + input: raw((p) => { + p.redactions.faces = [{ id: "f", target: { trackId: "track-x" }, mode: "pixelate" }]; + }), + }, + { + code: "unknown_template", + input: raw((p) => (p.intro = { templateId: "intro-none", durationMs: 1000 })), + path: "intro.templateId", + }, + { + code: "unknown_template", + input: raw((p) => (p.captions = { mode: "burned", presetId: "missing" })), + path: "captions.presetId", + }, + { + code: "range_out_of_asset", + input: raw((p) => (p.tracks.video[1].outMs = 12_000)), + path: "tracks.video[1].outMs", + }, + { + code: "range_out_of_asset", + input: raw((p) => (p.tracks.video[0].reframe = [{ tMs: 9000, cx: 0.5, cy: 0.5, scale: 1 }])), + }, + { + code: "range_empty", + input: raw((p) => (p.tracks.video[0].outMs = 0)), + }, + { + code: "range_empty", + input: raw( + (p) => (p.overlays = [{ id: "t", kind: "text", atMs: 0, durationMs: 0, text: "x" }]), + ), + }, + { + code: "overlap", + input: raw((p) => (p.tracks.video[1].atMs = 1500)), + path: "tracks.video[1].atMs", + }, + { + code: "overlap", + // A crossfade needs exactly its duration of overlap. + input: raw((p) => { + p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }]; + }), + }, + { + code: "overlap", + input: raw((p) => { + p.tracks.audio = [ + { id: "m1", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 0 }, + { id: "m2", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 2000 }, + ]; + }), + }, + { + code: "duration_below_min", + input: raw((p) => (p.output.durationHardMinMs = 5000)), + }, + { + code: "duration_below_min", + input: testEdl(), + context: { profile: testProfile({ duration: { hardMinMs: 6000, hardMaxMs: 60_000 } }) }, + }, + { + code: "duration_above_max", + input: raw((p) => (p.output.durationHardMaxMs = 3000)), + }, + { + code: "duration_above_max", + input: testEdl(), + context: { profile: testProfile({ duration: { hardMinMs: 0, hardMaxMs: 3000 } }) }, + }, + { + code: "overlay_outside_safe_zone", + input: raw((p) => { + p.overlays = [ + { + id: "t", + kind: "text", + atMs: 0, + durationMs: 1000, + text: "Hi", + box: { x: 0.1, y: 0.85, w: 0.5, h: 0.1 }, + }, + ]; + }), + path: "overlays[0].box", + }, + { + code: "paid_only_segment", + input: testEdl(), + context: { paidOnly: [{ assetId: "asset-a", startMs: 6500, endMs: 9000 }] }, + path: "tracks.video[1]", + }, + { + code: "paid_only_segment", + input: testEdl({ output: { ...testEdl().output, profileId: "example-paid-video" } }), + context: { + profile: testProfile({ id: "example-paid-video", kind: "paid_video" }), + paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 100 }], + goal: "teaser", + }, + }, + { + code: "profile_mismatch", + input: raw((p) => (p.output.profileId = "other-profile")), + }, + { + code: "profile_mismatch", + input: raw((p) => { + p.output.width = 1920; + p.output.height = 1080; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "vo-1", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-1", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + context: { profile: testProfile({ audio: { music: "platform_library_only" } }) }, + }, + ]; + + it.each(cases)("rejects $code ($path)", ({ code, input, context, path }) => { + const result = validateEdl(input, testContext(context)); + expect(result.ok).toBe(false); + if (result.ok) return; + for (const error of result.errors) expect(EdlErrorSchema.safeParse(error).success).toBe(true); + const match = result.errors.find((e) => e.code === code); + expect(match, JSON.stringify(result.errors)).toBeDefined(); + if (path) expect(match!.path).toBe(path); + }); + + it("covers every error code", () => { + const covered = new Set(cases.map((c) => c.code)); + expect([...covered].sort()).toEqual([...EdlErrorSchema.shape.code.options].sort()); + }); + + it("allows paid-only material in a full release for a paid profile and any music where allowed", () => { + const paid = testProfile({ + id: "example-paid-video", + kind: "paid_video", + audio: { music: "any" }, + }); + const plan = raw((p) => { + p.output.profileId = "example-paid-video"; + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }); + expect( + codes( + plan, + testContext({ profile: paid, paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 9000 }] }), + ), + ).toEqual([]); + }); + + it("accepts photos without a duration and a crossfade with the right overlap", () => { + const plan = raw((p) => { + p.tracks.video[0].assetId = "photo-1"; + p.tracks.video[1].atMs = 1500; + p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }]; + }); + expect(codes(plan)).toEqual([]); + }); + + it("reports every failure at once", () => { + const plan = raw((p) => { + p.tracks.video[0].assetId = "missing"; + p.tracks.video[1].outMs = 4000; + p.redactions.marks = [{ markId: "mark-x", treatment: "cover" }]; + }); + expect(codes(plan)).toEqual(["unknown_asset", "range_empty", "unknown_mark"]); + }); +}); + +describe("timeline", () => { + const edl = testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1000 }, + outro: { templateId: "outro-simple", durationMs: 500 }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" }, + { + id: "c2", + assetId: "asset-b", + inMs: 1000, + outMs: 5000, + atMs: 1500, + speed: 2, + reframe: "none", + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "dip", durationMs: 500 }], + }); + + it("measures intro, speed, transition overlap and outro", () => { + expect(edlDurationMs(edl)).toBe(1000 + 1500 + 2000 + 500); + expect(edlDurationMs(testEdl())).toBe(4000); + }); + + it("maps output time back to clip and source time", () => { + expect(locateOutputTime(edl, 500)).toBeNull(); + expect(locateOutputTime(edl, 1000)).toEqual({ clipId: "c1", assetId: "asset-a", sourceMs: 0 }); + expect(locateOutputTime(edl, 2200)).toEqual({ + clipId: "c1", + assetId: "asset-a", + sourceMs: 1200, + }); + // Inside the transition both clips are visible; the incoming one wins. + expect(locateOutputTimeAll(edl, 2700).map((h) => h.clipId)).toEqual(["c1", "c2"]); + expect(locateOutputTime(edl, 2700)).toEqual({ + clipId: "c2", + assetId: "asset-b", + sourceMs: 1400, + }); + expect(locateOutputTime(edl, 4400)).toEqual({ + clipId: "c2", + assetId: "asset-b", + sourceMs: 4800, + }); + expect(locateOutputTime(edl, 4600)).toBeNull(); + }); +}); diff --git a/packages/media/src/gate.test.ts b/packages/media/src/gate.test.ts new file mode 100644 index 0000000..acb75e3 --- /dev/null +++ b/packages/media/src/gate.test.ts @@ -0,0 +1,531 @@ +import type { FfprobeReport, GateReport } from "@rakazo/contracts"; +import { GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { edlDurationMs } from "./edl.js"; +import type { GateInput } from "./gate.js"; +import { + checkCopy, + checkTechnical, + evaluateGate, + findMetadataLeaks, + METADATA_ALLOWED_TAGS, + planAutoFix, +} from "./gate.js"; +import { testEdl, testPaidProfile, testProfile } from "./test-fixtures.js"; + +const cleanTags = { + "format.major_brand": "isom", + "format.minor_version": "512", + "format.compatible_brands": "isomiso2avc1mp41", + "format.title": "Example", + "stream.0.language": "und", + "stream.0.handler_name": "VideoHandler", + "stream.0.vendor_id": "[0][0][0][0]", + "stream.0.encoder": "Lavc libx264", + "stream.1.handler_name": "SoundHandler", +}; + +function probe(overrides: Partial = {}): FfprobeReport { + return { + durationMs: 4000, + width: 1080, + height: 1920, + fps: 30, + videoCodec: "h264", + audioCodec: "aac", + sizeBytes: 2_000_000, + integratedLufs: -14.3, + truePeakDb: -1.2, + tags: cleanTags, + ...overrides, + }; +} + +function gateInput(overrides: Partial = {}): GateInput { + return { + profile: testProfile(), + edl: testEdl(), + ffprobe: probe(), + content: [ + { tMs: 0, scores: { explicit_nudity: 0.01 } }, + { tMs: 500, scores: { explicit_nudity: 0.02, violence: 0.1 } }, + ], + faces: [], + trackingGaps: [], + marks: [], + people: [], + copy: { title: "A day out", caption: "Short caption #one", tags: ["two"] }, + aiAlterations: [], + bundleHasDisclosure: false, + watermarkBox: { x: 0.7, y: 0.12, w: 0.15, h: 0.05 }, + sampledFrames: 8, + autoFixRound: 0, + now: new Date("2026-10-01T12:00:00Z"), + ...overrides, + }; +} + +const blockCodes = (report: GateReport) => + report.issues.filter((i) => i.severity === "block").map((i) => i.code); + +describe("metadata", () => { + it("allows only harmless container tags", () => { + expect(METADATA_ALLOWED_TAGS).toContain("title"); + expect(findMetadataLeaks(cleanTags)).toEqual([]); + expect( + findMetadataLeaks({ + ...cleanTags, + "format.creation_time": "2026-01-01T00:00:00Z", + "format.location": "+00.0000+000.0000/", + "format.com.apple.quicktime.make": "Example", + "format.com.apple.quicktime.location.ISO6709": "+00+000/", + "format.make": "Example", + "format.model": "Example", + "format.software": "Example 1.0", + "stream.0.creation_time": "2026-01-01T00:00:00Z", + GPSLatitude: "0", + "format.encoder": "Lavf60.16.100", + "stream:0:encoder": "x264 core 164", + }), + ).toEqual([ + "GPSLatitude", + "format.com.apple.quicktime.location.ISO6709", + "format.com.apple.quicktime.make", + "format.creation_time", + "format.encoder", + "format.location", + "format.make", + "format.model", + "format.software", + "stream.0.creation_time", + "stream:0:encoder", + ]); + expect(findMetadataLeaks({ title: "x", major_brand: "isom" })).toEqual([]); + }); +}); + +describe("evaluateGate", () => { + it("passes a clean render and produces a schema-valid report", () => { + const report = evaluateGate(gateInput()); + expect(report.issues).toEqual([]); + expect(report.passed).toBe(true); + expect(GateReportSchema.parse(report)).toEqual(report); + expect(report.checkedAt).toBe("2026-10-01T12:00:00.000Z"); + }); + + it("blocks technical problems", () => { + const edl = testEdl(); + expect(checkTechnical(testProfile(), edl, probe())).toEqual([]); + const cases: Array<[Partial, string]> = [ + [{ width: 1920, height: 1080 }, "resolution_mismatch"], + [{ width: 1920, height: 1080 }, "aspect_not_allowed"], + [{ fps: 25 }, "fps_mismatch"], + [{ durationMs: 70_000 }, "duration_too_long"], + [{ durationMs: 500 }, "duration_too_short"], + [{ sizeBytes: 600_000_000 }, "file_too_large"], + [{ videoCodec: "vp9" }, "codec_mismatch"], + [{ integratedLufs: -20 }, "loudness_off_target"], + [{ integratedLufs: null }, "loudness_unmeasured"], + [{ truePeakDb: 0 }, "true_peak_exceeded"], + [{ tags: { ...cleanTags, "format.creation_time": "2020" } }, "metadata_leak"], + ]; + for (const [patch, code] of cases) { + const report = evaluateGate(gateInput({ ffprobe: probe(patch) })); + expect(report.passed, code).toBe(false); + expect(blockCodes(report), code).toContain(code); + } + const drift = checkTechnical(testProfile(), edl, probe({ durationMs: 4500 })); + expect(drift).toEqual([expect.objectContaining({ code: "duration_drift", severity: "warn" })]); + expect( + checkTechnical(testProfile(), edl, probe({ audioCodec: null, integratedLufs: null })), + ).toEqual([]); + }); + + it("blocks classifier hits with timestamps, category and score", () => { + const report = evaluateGate( + gateInput({ + content: [ + { tMs: 0, scores: { explicit_nudity: 0.05 } }, + { tMs: 1500, scores: { explicit_nudity: 0.4 } }, + { tMs: 2000, scores: { explicit_nudity: 0.6 } }, + { tMs: 3500, scores: { violence: 0.9, drugs: 1 } }, + ], + }), + ); + expect(report.passed).toBe(false); + expect(report.issues).toEqual([ + expect.objectContaining({ + check: "content", + code: "content_threshold", + category: "explicit_nudity", + atMs: 1500, + endMs: 2000, + score: 0.6, + }), + expect.objectContaining({ category: "violence", atMs: 3500, severity: "block" }), + ]); + }); + + it("only blocks above the ratio when the profile says so", () => { + const profile = testProfile({ + content: { thresholds: { violence: 0.5 }, blockOn: "ratio", blockRatio: 0.25 }, + }); + const samples = Array.from({ length: 10 }, (_, i) => ({ + tMs: i * 500, + scores: { violence: i < 2 ? 0.9 : 0 }, + })); + const below = evaluateGate(gateInput({ profile, content: samples })); + expect(below.passed).toBe(true); + expect(below.issues[0]).toMatchObject({ severity: "warn", category: "violence" }); + const above = samples.map((s, i) => ({ ...s, scores: { violence: i < 4 ? 0.9 : 0 } })); + expect(evaluateGate(gateInput({ profile, content: above })).passed).toBe(false); + }); + + it("blocks anonymization gaps, uncovered and unknown faces, and visible marks", () => { + const people = [ + { id: "person-1", anonymizeByDefault: "blur", consentRecordedAt: null, ageVerifiedAt: null }, + { id: "person-2", anonymizeByDefault: "none", consentRecordedAt: null, ageVerifiedAt: null }, + ]; + const box = { x: 0.4, y: 0.2, w: 0.1, h: 0.1 }; + const report = evaluateGate( + gateInput({ + people, + faces: [ + { tMs: 100, box, personId: "person-1", coverage: 1 }, + { tMs: 200, box, personId: "person-1", coverage: 0.9 }, + { tMs: 300, box, personId: "person-2", coverage: 0 }, + { tMs: 400, box, personId: null, coverage: 0 }, + ], + trackingGaps: [ + { startMs: 1000, endMs: 1400, personId: "person-1" }, + { startMs: 2000, endMs: 2600, personId: "person-1" }, + ], + marks: [ + { markId: "mark-1", tMs: 100, residualVisibility: 0.01 }, + { markId: "mark-1", tMs: 300, residualVisibility: 0.3 }, + ], + }), + ); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([ + expect.objectContaining({ code: "face_not_covered", personId: "person-1", atMs: 200 }), + expect.objectContaining({ code: "unknown_face", atMs: 400 }), + expect.objectContaining({ + code: "tracking_gap", + atMs: 2000, + endMs: 2600, + personId: "person-1", + }), + expect.objectContaining({ code: "mark_visible", markId: "mark-1", atMs: 300 }), + ]); + // Without protected people an unmatched face is not an anonymization issue. + const open = evaluateGate(gateInput({ faces: [{ tMs: 0, box, personId: null, coverage: 0 }] })); + expect(open.passed).toBe(true); + }); + + it("requires consent and age verification only where the profile does", () => { + const people = [ + { + id: "p-ok", + anonymizeByDefault: "none", + consentRecordedAt: "2026-01-01T00:00:00Z", + ageVerifiedAt: "2026-01-01T00:00:00Z", + }, + { + id: "p-no-consent", + anonymizeByDefault: "none", + consentRecordedAt: null, + ageVerifiedAt: "2026-01-01T00:00:00Z", + }, + { + id: "p-no-age", + anonymizeByDefault: "none", + consentRecordedAt: "2026-01-01T00:00:00Z", + ageVerifiedAt: null, + }, + ]; + const faces = [{ tMs: 1000, box: { x: 0, y: 0, w: 0.1, h: 0.1 }, personId: null, coverage: 0 }]; + expect(evaluateGate(gateInput({ people, faces })).passed).toBe(true); + const profile = testPaidProfile(); + const edl = testEdl({ + output: { ...testEdl().output, profileId: profile.id, width: 1920, height: 1080 }, + }); + const report = evaluateGate( + gateInput({ profile, edl, people, faces, ffprobe: probe({ width: 1920, height: 1080 }) }), + ); + expect(report.issues.filter((i) => i.check === "consent")).toEqual([ + expect.objectContaining({ code: "age_unverified", personId: "p-no-age" }), + expect.objectContaining({ code: "consent_missing", personId: "p-no-consent" }), + expect.objectContaining({ code: "unknown_performer", atMs: 1000 }), + ]); + expect(report.passed).toBe(false); + }); + + it("requires the AI label when the profile does and warns about the watermark", () => { + const report = evaluateGate( + gateInput({ + aiAlterations: ["synthetic_identity"], + watermarkBox: { x: 0.9, y: 0.01, w: 0.1, h: 0.05 }, + }), + ); + expect(report.issues).toEqual([ + expect.objectContaining({ check: "disclosure", code: "ai_label_missing", severity: "block" }), + expect.objectContaining({ + check: "brand", + code: "watermark_outside_safe_zone", + severity: "warn", + }), + ]); + expect( + evaluateGate(gateInput({ aiAlterations: ["inpaint"], bundleHasDisclosure: true })).passed, + ).toBe(true); + const edl = testEdl({ metadata: { aiAltered: true, aiAlterations: ["voice_disguise"] } }); + expect(blockCodes(evaluateGate(gateInput({ edl })))).toEqual(["ai_label_missing"]); + const relaxed = testProfile({ disclosure: { aiAlteredLabelRequired: false } }); + expect(evaluateGate(gateInput({ profile: relaxed, edl })).passed).toBe(true); + }); +}); + +describe("checkCopy", () => { + const profile = testProfile({ + text: { + titleMaxChars: 20, + captionMaxChars: 60, + hashtagLimit: 2, + linkPolicy: "bio_only", + bannedPatterns: [ + { pattern: "free\\s+money", reason: "misleading claim" }, + { pattern: "([a-z", reason: "broken pattern" }, + ], + }, + }); + + it("checks banned patterns case-insensitively and reports invalid ones without crashing", () => { + const issues = checkCopy(profile, { + title: "FREE Money", + caption: null, + tags: ["free money"], + }); + expect(issues.map((i) => [i.code, i.message])).toEqual([ + ["banned_pattern", "title: misleading claim"], + ["banned_pattern", "tags[0]: misleading claim"], + ["invalid_banned_pattern", "profile pattern cannot be checked: broken pattern"], + ]); + }); + + it("checks lengths, hashtags and links", () => { + const codes = (copy: { title: string | null; caption: string | null; tags: string[] }) => + checkCopy(profile, copy) + .map((i) => i.code) + .filter((c) => c !== "invalid_banned_pattern"); + expect(codes({ title: "x".repeat(21), caption: "y".repeat(61), tags: [] })).toEqual([ + "title_too_long", + "caption_too_long", + ]); + expect(codes({ title: null, caption: "#a #b", tags: ["#c"] })).toEqual(["too_many_hashtags"]); + expect(codes({ title: null, caption: "#a #A", tags: ["a"] })).toEqual([]); + expect(codes({ title: null, caption: "see https://example.com/x", tags: [] })).toEqual([ + "link_bio_only", + ]); + expect(codes({ title: null, caption: "visit example.shop today", tags: [] })).toEqual([ + "link_bio_only", + ]); + expect(codes({ title: "Link in bio", caption: "Ready? Go.", tags: [] })).toEqual([]); + const forbidden = testProfile({ + text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "forbidden" }, + }); + expect( + checkCopy(forbidden, { title: "www.example.com", caption: null, tags: [] })[0]!.code, + ).toBe("link_forbidden"); + const allowed = testProfile({ + text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "allowed" }, + }); + expect(checkCopy(allowed, { title: null, caption: "https://example.com", tags: [] })).toEqual( + [], + ); + }); +}); + +describe("planAutoFix", () => { + const edl = testEdl({ + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 0, + outMs: 2000, + atMs: 0, + reframe: "auto", + segmentId: "s1", + }, + { + id: "c2", + assetId: "asset-a", + inMs: 5000, + outMs: 7000, + atMs: 2000, + reframe: "auto", + segmentId: "s2", + }, + { + id: "c3", + assetId: "asset-a", + inMs: 8000, + outMs: 10_000, + atMs: 4000, + reframe: "auto", + segmentId: "s3", + }, + ], + }, + transitions: [{ fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }], + overlays: [ + { id: "wm", kind: "watermark", atMs: 0, durationMs: 6000 }, + { id: "cta", kind: "cta_end_card", atMs: 4500, durationMs: 1500, text: "Link in bio" }, + ], + redactions: { + faces: [{ id: "f1", target: { personId: "person-1" }, mode: "blur", paddingPct: 20 }], + marks: [{ markId: "mark-1", treatment: "inpaint" }], + }, + metadata: { aiAltered: true, aiAlterations: ["inpaint"] }, + }); + const report = (issues: GateReport["issues"], round = 0): GateReport => ({ + profileId: "example-short-vertical", + passed: !issues.some((i) => i.severity === "block"), + checkedAt: "2026-10-01T12:00:00.000Z", + sampledFrames: 12, + issues, + autoFixRound: round, + }); + const contentHit = (atMs: number, endMs = atMs) => + ({ + check: "content", + severity: "block", + code: "content_threshold", + message: "violence", + atMs, + endMs, + category: "violence", + score: 0.9, + }) as const; + + const assertNoNewOverlays = (fixed: typeof edl) => { + const before = new Map(edl.overlays.map((o) => [o.id, o])); + for (const overlay of fixed.overlays) { + const original = before.get(overlay.id); + expect(original, `overlay ${overlay.id} was added`).toBeDefined(); + expect(overlay.kind).toBe(original!.kind); + expect(overlay.text).toBe(original!.text); + expect(overlay.durationMs).toBeLessThanOrEqual(original!.durationMs); + } + expect(fixed.overlays.length).toBeLessThanOrEqual(edl.overlays.length); + }; + + it("swaps an offending clip for a clean alternative segment", () => { + const fix = planAutoFix(edl, report([contentHit(2500, 3000)]), [ + { segmentId: "s2", assetId: "asset-a", startMs: 5000, endMs: 7000 }, + { segmentId: "s9", assetId: "asset-a", startMs: 5500, endMs: 9000 }, + { segmentId: "s8", assetId: "asset-b", startMs: 1000, endMs: 4000 }, + ])!; + expect(fix.changes).toEqual(["clip c2: swapped to segment s8 (content_threshold)"]); + expect(fix.edl.tracks.video.find((c) => c.id === "c2")).toMatchObject({ + assetId: "asset-b", + inMs: 1000, + outMs: 3000, + segmentId: "s8", + atMs: 2000, + }); + expect(edlDurationMs(fix.edl)).toBe(6000); + assertNoNewOverlays(fix.edl); + }); + + it("trims or drops the clip when no alternative fits, and repacks the timeline", () => { + const trimmed = planAutoFix(edl, report([contentHit(3800)]), [])!; + expect(trimmed.changes).toEqual(["clip c2: trimmed the end at 6300 ms (content_threshold)"]); + expect(trimmed.edl.tracks.video.map((c) => [c.id, c.atMs, c.inMs, c.outMs])).toEqual([ + ["c1", 0, 0, 2000], + ["c2", 2000, 5000, 6300], + ["c3", 3300, 8000, 10_000], + ]); + assertNoNewOverlays(trimmed.edl); + expect(trimmed.edl.overlays.find((o) => o.id === "wm")!.durationMs).toBe(5300); + + const dropped = planAutoFix(edl, report([contentHit(2300, 3700)]), [])!; + expect(dropped.changes).toEqual(["clip c2: removed (content_threshold)"]); + expect(dropped.edl.tracks.video.map((c) => c.id)).toEqual(["c1", "c3"]); + expect(dropped.edl.transitions).toEqual([]); + assertNoNewOverlays(dropped.edl); + }); + + it("cuts tracking gaps, widens face redactions, adds a blur for unknown faces, and covers marks", () => { + const fix = planAutoFix( + edl, + report([ + { + check: "anonymization", + severity: "block", + code: "face_not_covered", + message: "", + atMs: 100, + personId: "person-1", + }, + { check: "anonymization", severity: "block", code: "unknown_face", message: "", atMs: 300 }, + { + check: "anonymization", + severity: "block", + code: "mark_visible", + message: "", + atMs: 300, + markId: "mark-1", + }, + { + check: "anonymization", + severity: "block", + code: "tracking_gap", + message: "", + atMs: 4100, + endMs: 4800, + }, + ]), + [], + )!; + expect(fix.edl.redactions.faces).toEqual([ + expect.objectContaining({ id: "f1", paddingPct: 45 }), + expect.objectContaining({ + id: "fix-face-unassigned", + target: { unassignedFaces: true }, + mode: "blur", + }), + ]); + expect(fix.edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "cover" }]); + expect(fix.edl.metadata.aiAlterations).toEqual([]); + expect(fix.edl.tracks.video.find((c) => c.id === "c3")).toMatchObject({ inMs: 9300 }); + assertNoNewOverlays(fix.edl); + }); + + it("stops at the round limit, on passing reports, and when nothing safe applies", () => { + expect(planAutoFix(edl, report([contentHit(2500)], MAX_AUTO_FIX_ROUNDS), [])).toBeNull(); + expect(planAutoFix(edl, report([]), [])).toBeNull(); + const textOnly = report([ + { check: "text", severity: "block", code: "banned_pattern", message: "" }, + ]); + expect(planAutoFix(edl, textOnly, [])).toBeNull(); + const consent = report([ + { check: "consent", severity: "block", code: "consent_missing", message: "", personId: "p" }, + ]); + expect(planAutoFix(edl, consent, [])).toBeNull(); + }); + + it("runs at most three rounds in a loop", () => { + let current = edl; + let rounds = 0; + for (let round = 0; round < 10; round++) { + const fix = planAutoFix(current, report([contentHit(500)], round), []); + if (!fix) break; + current = fix.edl; + rounds += 1; + assertNoNewOverlays(current); + } + expect(rounds).toBe(MAX_AUTO_FIX_ROUNDS); + }); +}); diff --git a/packages/media/src/marketing.test.ts b/packages/media/src/marketing.test.ts new file mode 100644 index 0000000..f2d43fc --- /dev/null +++ b/packages/media/src/marketing.test.ts @@ -0,0 +1,320 @@ +import type { MetricCsvMapping, MotionTemplateInfo } from "@rakazo/contracts"; +import { MetricCsvMappingSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { + buildTrackingUrl, + contrastRatio, + introFits, + normalizeHashtags, + parseCsvRecords, + parseMetricsCsv, + parseMoneyToCents, + proposeIntroVariants, + uploadChecklist, + validateBrandKit, +} from "./marketing.js"; +import { testKit, testPaidProfile, testProfile } from "./test-fixtures.js"; + +const utm = { source: "example-short-vertical", medium: "short", campaign: "spring launch" }; + +describe("buildTrackingUrl", () => { + it("appends UTM parameters and keeps the existing query and fragment", () => { + expect(buildTrackingUrl("https://example.com/shop", utm)).toBe( + "https://example.com/shop?utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch", + ); + expect( + buildTrackingUrl("https://example.com/p?ref=a%20b&x=1&utm_source=old#top", { + ...utm, + content: "thumb-2", + term: "ä&b", + }), + ).toBe( + "https://example.com/p?ref=a%20b&x=1&utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch&utm_content=thumb-2&utm_term=%C3%A4%26b#top", + ); + expect(buildTrackingUrl("http://example.com", utm)).toMatch( + /^http:\/\/example\.com\/\?utm_source=/, + ); + }); + + it("rejects non-http schemes and invalid URLs", () => { + expect(() => buildTrackingUrl("javascript:alert(1)", utm)).toThrow(/http/); + expect(() => buildTrackingUrl("ftp://example.com", utm)).toThrow(/http/); + expect(() => buildTrackingUrl("not a url", utm)).toThrow(/valid/); + }); +}); + +describe("normalizeHashtags", () => { + it("normalizes, dedupes and limits", () => { + expect( + normalizeHashtags(["#Summer", "summer", "##new day", "über-cool!", "", "#"], 10), + ).toEqual(["#Summer", "#newday", "#übercool"]); + expect(normalizeHashtags(["a", "b", "c"], 2)).toEqual(["#a", "#b"]); + expect(normalizeHashtags(["a"], 0)).toEqual([]); + }); +}); + +describe("CSV", () => { + const mapping = (overrides: Partial = {}): MetricCsvMapping => + MetricCsvMappingSchema.parse({ + name: "Example", + columns: { + publication: "Link", + observedAt: "Date", + views: "Views", + clicks: "Clicks", + conversions: "Sales", + revenue: "Revenue", + }, + publicationKey: "tracking_url", + ...overrides, + }); + + it("parses RFC 4180 records with quotes, delimiters and newlines in fields", () => { + expect(parseCsvRecords('a,"b,c","say ""hi""\nthere"\r\n1,2,3\n', ",")).toEqual([ + ["a", "b,c", 'say "hi"\nthere'], + ["1", "2", "3"], + ]); + expect(parseCsvRecords("x;y\n", ";")).toEqual([["x", "y"]]); + expect(parseCsvRecords("", ",")).toEqual([]); + }); + + it("imports metrics with BOM, comma delimiter and major-unit revenue", () => { + const csv = + 'Link,Date,Views,Clicks,Sales,Revenue\n"https://example.com/?utm_content=a,b",2026-09-01,"1,200",30,2,"19.98"\n'; + const result = parseMetricsCsv(csv, mapping()); + expect(result.skipped).toEqual([]); + expect(result.rows).toEqual([ + { + row: 2, + publicationKey: "https://example.com/?utm_content=a,b", + observedAt: new Date("2026-09-01T00:00:00Z"), + views: 1200, + clicks: 30, + conversions: 2, + revenueCents: 1998, + currency: "EUR", + }, + ]); + }); + + it("handles semicolons, tabs, decimal commas, currencies and skipped rows", () => { + const semicolon = [ + "link;date;views;clicks;sales;revenue;currency", + "pub-1;01.09.2026;1.500;12;1;1.234,50 €;", + "pub-2;2026/09/02;10;1;0;12,5;usd", + "pub-3;31.02.2026;1;0;0;0;", + "pub-4;2026-09-03;ten;0;0;0;", + ";2026-09-03;1;0;0;0;", + "pub-5;2026-09-04;1;0;0;-5;", + "pub-6;2026-09-04;1;0;0;1,2,3;", + "pub-7;2026-09-04;1;0;0;5;EURO", + ";;;;;;", + "pub-8;2026-09-05T10:30:00Z;7;;;$3.10;", + ].join("\r\n"); + const result = parseMetricsCsv( + semicolon, + mapping({ + delimiter: ";", + columns: { + publication: "LINK", + observedAt: "date", + views: "views", + clicks: "clicks", + conversions: "sales", + revenue: "revenue", + currency: "currency", + }, + }), + ); + expect(result.rows.map((r) => [r.publicationKey, r.views, r.revenueCents, r.currency])).toEqual( + [ + ["pub-1", 1500, 123_450, "EUR"], + ["pub-2", 10, 1250, "USD"], + ["pub-8", 7, 310, "USD"], + ], + ); + expect(result.rows[2]!.observedAt).toEqual(new Date("2026-09-05T10:30:00Z")); + expect(result.skipped).toEqual([ + { row: 4, reason: "date is not recognized" }, + { row: 5, reason: "views is not a whole number" }, + { row: 6, reason: "publication is empty" }, + { row: 7, reason: "revenue is negative" }, + { row: 8, reason: "revenue is not a number" }, + { row: 9, reason: "currency is not a 3-letter code" }, + ]); + + const tabs = parseMetricsCsv( + "Link\tDate\tRevenue\nx\t2026-09-01\t250\n", + mapping({ + delimiter: "\t", + revenueUnit: "cents", + columns: { publication: "Link", observedAt: "Date", revenue: "Revenue" }, + }), + ); + expect(tabs.rows[0]).toMatchObject({ views: 0, clicks: 0, conversions: 0, revenueCents: 250 }); + }); + + it("reports missing columns and empty files", () => { + expect(parseMetricsCsv("Other,Date\n", mapping())).toEqual({ + rows: [], + skipped: [{ row: 1, reason: "missing columns: Link, Views, Clicks, Sales, Revenue" }], + }); + expect(parseMetricsCsv("", mapping()).skipped).toEqual([{ row: 1, reason: "file is empty" }]); + }); + + it("converts money in either notation to cents", () => { + expect(parseMoneyToCents("1,234.56")).toEqual({ cents: 123_456, currency: null }); + expect(parseMoneyToCents("1.234,56")).toEqual({ cents: 123_456, currency: null }); + expect(parseMoneyToCents("1,234")).toEqual({ cents: 123_400, currency: null }); + expect(parseMoneyToCents("0.995")).toEqual({ cents: 100, currency: null }); + expect(parseMoneyToCents("2.5")).toEqual({ cents: 250, currency: null }); + expect(parseMoneyToCents("CHF 7")).toEqual({ cents: 700, currency: "CHF" }); + expect(parseMoneyToCents("12.345.678,9")).toEqual({ cents: 1_234_567_890, currency: null }); + expect(parseMoneyToCents("1,5,0")).toBeNull(); + }); +}); + +describe("intro", () => { + const templates: MotionTemplateInfo[] = [ + { + id: "intro-glitch", + kind: "intro", + name: "Glitch energy", + minDurationMs: 1000, + maxDurationMs: 5000, + params: [{ name: "primaryColor", type: "color", required: true }], + }, + { + id: "intro-elegant", + kind: "intro", + name: "Elegant fade", + minDurationMs: 2000, + maxDurationMs: 3000, + params: [ + { name: "brandName", type: "string", required: true }, + { name: "sound", type: "string", required: false }, + ], + }, + { + id: "intro-kinetic", + kind: "intro", + name: "Kinetic type", + minDurationMs: 1500, + maxDurationMs: 4000, + params: [], + }, + { + id: "intro-logo", + kind: "intro", + name: "Minimal logo reveal", + minDurationMs: 1500, + maxDurationMs: 4000, + params: [{ name: "logoArtifactId", type: "string", required: true }], + }, + { + id: "outro-a", + kind: "outro", + name: "Outro", + minDurationMs: 1000, + maxDurationMs: 3000, + params: [], + }, + ]; + + it("keeps shorts hook-first and allows short intros on long-form profiles", () => { + expect(introFits(testProfile(), 1500)).toBe(false); + expect(introFits(testProfile(), 0)).toBe(true); + expect(introFits(testPaidProfile(), 3000)).toBe(true); + expect(introFits(testPaidProfile(), 5000)).toBe(false); + }); + + it("proposes three distinct variants ordered by tone", () => { + const kit = testKit({ tone: "calm, premium", logoArtifactIds: ["logo-1"] }); + const variants = proposeIntroVariants(kit, templates, 3); + expect(variants.map((v) => v.template.templateId)).toEqual([ + "intro-elegant", + "intro-logo", + "intro-glitch", + ]); + expect(new Set(variants.map((v) => v.template.templateId)).size).toBe(3); + for (const variant of variants) { + expect(variant.template.durationMs).toBeGreaterThanOrEqual(1500); + expect(variant.template.durationMs).toBeLessThanOrEqual(4000); + } + expect(variants[0]!.template.params).toEqual({ brandName: "Studio Example", sound: "music-1" }); + expect(variants[0]!.rationale).toContain('matches the tone "calm, premium"'); + expect(variants[1]!.template.params).toEqual({ logoArtifactId: "logo-1" }); + expect(proposeIntroVariants(kit, templates, 3)).toEqual(variants); + const energetic = proposeIntroVariants(testKit({ tone: "bold and energetic" }), templates, 3); + expect(energetic[0]!.template).toMatchObject({ + templateId: "intro-glitch", + params: { primaryColor: "#112233" }, + }); + expect(proposeIntroVariants(kit, [], 3)).toEqual([]); + expect( + proposeIntroVariants(kit, templates.slice(0, 1), 3).map((v) => v.template.durationMs), + ).toEqual([2500, 3000, 1500]); + }); +}); + +describe("validateBrandKit", () => { + it("flags low contrast, duplicate presets, unlicensed music and fonts", () => { + expect(validateBrandKit(testKit())).toEqual([]); + expect(contrastRatio("#FFFFFF", "#000000")).toBeCloseTo(21, 5); + const preset = testKit().captionPresets[0]!; + const warnings = validateBrandKit( + testKit({ + colors: { ...testKit().colors, text: "#777777", background: "#888888" }, + captionPresets: [preset, { ...preset, backgroundColor: "#EEEEEE" }], + musicBeds: [ + { artifactId: "m", title: "Borrowed", license: "unknown", commercialUse: false }, + ], + fonts: [{ family: "Nameless", artifactId: "f", role: "body" }], + }), + ); + expect(warnings).toEqual([ + "text on background has contrast 1.3:1, below 4.5:1", + 'caption preset id "clean" is used twice', + 'caption preset "clean" has contrast 1.2:1, below 4.5:1', + 'music bed "Borrowed" is not licensed for commercial use', + 'font "Nameless" has no license note', + ]); + }); +}); + +describe("uploadChecklist", () => { + it("lists the AI label, age restriction, link policy and captions where they apply", () => { + const paid = uploadChecklist({ + profile: testPaidProfile(), + aiAltered: true, + hasCaptions: true, + trackingUrl: "https://example.com/?utm_source=x", + }); + expect(paid).toContain("# Upload checklist: Example paid video"); + expect(paid).toContain("AI-generated or altered content label"); + expect(paid).toContain("age-restricted"); + expect(paid).toContain("Add the tracking link: "); + expect(paid).toContain("SRT or VTT"); + + const short = uploadChecklist({ + profile: testProfile({ captions: { recommended: true, required: true } }), + aiAltered: false, + hasCaptions: false, + trackingUrl: "https://example.com/", + }); + expect(short).not.toContain("AI-generated"); + expect(short).not.toContain("age-restricted"); + expect(short).toContain("in the profile bio, not in the caption"); + expect(short).toContain("Captions are required"); + expect(short).toContain("at most 3 hashtags"); + expect(short.split("\n").filter((l) => l.startsWith("- [ ] "))).toHaveLength(4); + + const relaxed = uploadChecklist({ + profile: testProfile({ disclosure: { aiAlteredLabelRequired: false } }), + aiAltered: true, + hasCaptions: true, + trackingUrl: null, + }); + expect(relaxed).not.toContain("AI-generated"); + }); +}); diff --git a/packages/media/src/teaser.test.ts b/packages/media/src/teaser.test.ts new file mode 100644 index 0000000..e2f218c --- /dev/null +++ b/packages/media/src/teaser.test.ts @@ -0,0 +1,334 @@ +import { + MediaProjectGoalSchema, + TEASER_FEATURES, + TEASER_LEARNING_MIN_SAMPLES, +} from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { validateEdl } from "./edl.js"; +import type { ScoredSegmentInput, SegmentFeatures } from "./teaser.js"; +import { + autoPlanEdl, + defaultTeaserWeights, + fitTeaserWeights, + rankThumbnailFrames, + scoreSegment, + selectTeaserSegments, + speechHookScore, +} from "./teaser.js"; +import { testContext, testKit, testPaidProfile, testProfile } from "./test-fixtures.js"; + +function segment( + id: string, + assetId: string, + startMs: number, + endMs: number, + features: SegmentFeatures, + extra: Partial = {}, +): ScoredSegmentInput { + return { + id, + assetId, + startMs, + endMs, + capturedAt: null, + features, + paidOnly: false, + transcript: null, + ...extra, + }; +} + +/** One 30 s asset building up to a climax at 20–24 s, plus a second asset. */ +const segments: ScoredSegmentInput[] = [ + segment("s1", "asset-a", 0, 4000, { motion: 0.2, sharpness: 0.8 }), + segment( + "s2", + "asset-a", + 4000, + 8000, + { motion: 0.3, audioPeak: 0.3 }, + { transcript: "Wait, you won't believe what happens next?" }, + ), + segment("s3", "asset-a", 8000, 12_000, { motion: 0.5, faces: 0.6 }), + segment("s4", "asset-a", 12_000, 16_000, { motion: 0.6, audioPeak: 0.5 }), + segment("s5", "asset-a", 16_000, 20_000, { motion: 0.7, novelty: 0.6 }, { paidOnly: true }), + segment("s6", "asset-a", 20_000, 24_000, { + motion: 1, + audioPeak: 1, + faces: 1, + novelty: 1, + aesthetics: 1, + }), + segment("s7", "asset-a", 24_000, 28_000, { motion: 0.4 }), + segment("s8", "asset-b", 0, 3000, { motion: 0.5, aesthetics: 0.7 }), +]; + +describe("weights and scores", () => { + it("defines every feature for every goal", () => { + for (const goal of MediaProjectGoalSchema.options) { + const weights = defaultTeaserWeights(goal); + expect(Object.keys(weights).sort()).toEqual([...TEASER_FEATURES].sort()); + expect(Object.values(weights).reduce((a, b) => a + b, 0)).toBeCloseTo(1, 6); + } + expect(defaultTeaserWeights("teaser").speechHook).toBeGreaterThan( + defaultTeaserWeights("highlight").speechHook, + ); + }); + + it("scores as a weighted mean of clamped features", () => { + const weights = defaultTeaserWeights("teaser"); + expect(scoreSegment({}, weights)).toBe(0); + const all = Object.fromEntries(TEASER_FEATURES.map((f) => [f, 1])); + expect(scoreSegment(all, weights)).toBeCloseTo(1, 6); + expect(scoreSegment({ motion: 5 }, weights)).toBeCloseTo(weights.motion, 6); + expect(scoreSegment({ motion: Number.NaN }, weights)).toBe(0); + expect(scoreSegment({ motion: 1 }, { ...weights, motion: 0 })).toBe(0); + }); + + it("detects German and English speech hooks", () => { + expect(speechHookScore("")).toBe(0); + expect(speechHookScore("the weather was fine")).toBe(0); + expect(speechHookScore("Wait, you won't believe this!", "en")).toBeGreaterThan(0.6); + expect(speechHookScore("Warum macht das niemand?", "de")).toBeGreaterThan(0.5); + expect(speechHookScore("Schau mal, 3 Tricks!", "de-DE")).toBeGreaterThan(0.6); + expect(speechHookScore("Here are 5 tips")).toBeGreaterThan(speechHookScore("here are tips")); + expect(speechHookScore("Wie geht das? Pass auf! 10 Sekunden, du!")).toBe(1); + }); +}); + +describe("selectTeaserSegments", () => { + const profile = testProfile(); + const select = (overrides: Partial[0]> = {}) => + selectTeaserSegments({ + segments, + profile, + goal: "teaser", + targetMs: 15_000, + weights: defaultTeaserWeights("teaser"), + endCardMs: 2000, + ...overrides, + }); + + it("opens with the hook inside the hook window", () => { + const selection = select(); + expect(selection.hookSegmentId).toBe("s2"); + expect(selection.clips[0]).toMatchObject({ segmentId: "s2", inMs: 4000 }); + expect(selection.notes[0]).toContain("hook window"); + }); + + it("never uses paid-only segments and ends before the payoff", () => { + const selection = select(); + const ids = selection.clips.map((c) => c.segmentId); + expect(ids).not.toContain("s5"); + expect(ids).not.toContain("s6"); + expect(ids).not.toContain("s7"); + const last = selection.clips.at(-1)!; + expect(last.assetId).toBe("asset-a"); + expect(last.outMs).toBeLessThanOrEqual(20_000); + expect(selection.notes.some((n) => n.startsWith("open loop"))).toBe(true); + }); + + it("builds tension and fits the duration bounds minus the end card", () => { + const selection = select(); + const total = selection.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0); + expect(total).toBeLessThanOrEqual(15_000 - 2000); + expect(total).toBeGreaterThanOrEqual(profile.duration.hardMinMs - 2000); + for (const clip of selection.clips) expect(clip.outMs).toBeGreaterThan(clip.inMs); + const tight = select({ targetMs: 5000 }); + expect(tight.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0)).toBeLessThanOrEqual(3000); + }); + + it("returns nothing when only paid-only material exists", () => { + const paid = segments.map((s) => ({ ...s, paidOnly: true })); + expect(select({ segments: paid })).toEqual({ + clips: [], + hookSegmentId: null, + notes: ["no usable segments"], + }); + }); + + it("is deterministic", () => { + expect(select({ segments: [...segments].reverse() })).toEqual(select()); + }); +}); + +describe("fitTeaserWeights", () => { + const prior = defaultTeaserWeights("teaser"); + const samples = (count: number) => + Array.from({ length: count }, (_, i) => { + const hook = (i % 10) / 9; + const motion = ((i * 7) % 10) / 9; + return { + features: { speechHook: hook, motion }, + views: 1000, + conversions: Math.round(1000 * (0.01 + 0.2 * hook)), + }; + }); + + it("stays off below the minimum sample count", () => { + expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES - 1), prior)).toBeNull(); + const withoutViews = [ + ...samples(TEASER_LEARNING_MIN_SAMPLES - 1), + { features: {}, views: 0, conversions: 0 }, + ]; + expect(fitTeaserWeights(withoutViews, prior)).toBeNull(); + }); + + it("shifts weight toward features that drive conversion, deterministically", () => { + const learned = fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)!; + expect(learned).not.toBeNull(); + expect(learned.speechHook).toBeGreaterThan(prior.speechHook); + expect(learned.motion).toBeLessThan(prior.motion); + expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)).toEqual(learned); + const more = fitTeaserWeights(samples(200), prior)!; + expect(more.speechHook).toBeGreaterThan(learned.speechHook); + }); +}); + +describe("autoPlanEdl", () => { + const assets = new Map([ + ["asset-a", { durationMs: 30_000, width: 1920, height: 1080, hasSpeech: true }], + ["asset-b", { durationMs: 8000, width: 1080, height: 1920, hasSpeech: false }], + ]); + const context = testContext({ + assets: new Map([ + ["asset-a", { kind: "video", durationMs: 30_000, width: 1920, height: 1080 }], + ["asset-b", { kind: "video", durationMs: 8000, width: 1080, height: 1920 }], + ]), + paidOnly: [{ assetId: "asset-a", startMs: 16_000, endMs: 20_000 }], + templateIds: new Set(["intro-minimal", "end-card-a"]), + goal: "teaser", + }); + + it("plans a valid teaser with hook first, captions, watermark, CTA and redactions", () => { + const kit = testKit({ + intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 }, + endCard: { templateId: "end-card-a", params: { style: "a" }, durationMs: 2000 }, + lutArtifactId: "lut-1", + }); + const edl = autoPlanEdl({ + goal: "teaser", + profile: testProfile(), + brandKit: kit, + segments, + weights: defaultTeaserWeights("teaser"), + assets, + people: [ + { id: "person-1", anonymizeByDefault: "synthetic_identity" }, + { id: "person-2", anonymizeByDefault: "none" }, + ], + marks: [ + { id: "mark-1", treatment: "inpaint", assetId: "asset-a", personId: null }, + { id: "mark-2", treatment: "blur", assetId: "asset-z", personId: null }, + { id: "mark-3", treatment: "none", assetId: "asset-a", personId: null }, + ], + ctaText: "Full video: link in bio", + }); + const result = validateEdl(edl, { ...context, brandKit: kit }); + expect(result.ok ? [] : result.errors).toEqual([]); + // Short vertical profile: no intro, the hook comes first. + expect(edl.intro).toBeUndefined(); + expect(edl.tracks.video[0]!.segmentId).toBe("s2"); + expect(edl.tracks.video[0]!.reframe).toBe("auto"); + expect(edl.tracks.video[0]!.color?.useBrandLut).toBe(true); + expect(edl.outro).toMatchObject({ + templateId: "end-card-a", + params: { style: "a", cta: "Full video: link in bio" }, + }); + expect(edl.captions).toMatchObject({ mode: "both", presetId: "clean", maxCharsPerLine: 24 }); + expect(edl.overlays.map((o) => o.kind)).toEqual(["watermark"]); + expect(edl.redactions.faces.map((f) => [f.target, f.mode])).toEqual([ + [{ personId: "person-1" }, "synthetic_identity"], + [{ unassignedFaces: true }, "blur"], + ]); + expect(edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "inpaint" }]); + expect(edl.metadata).toMatchObject({ + aiAltered: true, + aiAlterations: ["synthetic_identity", "inpaint"], + }); + }); + + it("uses a CTA text card without an end-card template and keeps the intro on long-form profiles", () => { + const profile = testPaidProfile(); + const kit = testKit({ + intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 }, + watermark: undefined, + }); + const teaser = autoPlanEdl({ + goal: "teaser", + profile, + brandKit: kit, + segments, + weights: defaultTeaserWeights("teaser"), + assets, + people: [], + marks: [], + ctaText: "Watch the full video", + }); + expect(teaser.intro?.templateId).toBe("intro-minimal"); + expect(teaser.overlays).toEqual([ + expect.objectContaining({ kind: "cta_end_card", text: "Watch the full video" }), + ]); + expect(teaser.redactions.faces).toEqual([]); + expect(teaser.metadata.aiAltered).toBe(false); + const result = validateEdl(teaser, { ...context, profile, brandKit: kit }); + expect(result.ok ? [] : result.errors).toEqual([]); + }); + + it("plans a full release in capture order including paid-only material on paid profiles", () => { + const profile = testPaidProfile(); + const edl = autoPlanEdl({ + goal: "full_release", + profile, + brandKit: null, + segments, + weights: defaultTeaserWeights("full_release"), + assets, + people: [], + marks: [], + ctaText: null, + }); + expect(edl.tracks.video.map((c) => c.segmentId)).toEqual([ + "s1", + "s2", + "s3", + "s4", + "s5", + "s6", + "s7", + "s8", + ]); + expect(edl.tracks.video.at(-1)!.reframe).toBe("auto"); + expect(edl.tracks.video[0]!.reframe).toBe("none"); + const result = validateEdl(edl, { ...context, profile, brandKit: null, goal: "full_release" }); + expect(result.ok ? [] : result.errors).toEqual([]); + expect(() => + autoPlanEdl({ + goal: "teaser", + profile, + brandKit: null, + segments: [], + weights: defaultTeaserWeights("teaser"), + assets, + people: [], + marks: [], + ctaText: null, + }), + ).toThrow(/no usable segments/); + }); +}); + +describe("rankThumbnailFrames", () => { + it("returns the best frames spread over the video", () => { + const candidates = [ + { tMs: 0, sharpness: 0.5, face: 0, thirds: 0.5, contrast: 0.5 }, + { tMs: 1000, sharpness: 1, face: 1, thirds: 1, contrast: 1 }, + { tMs: 1200, sharpness: 1, face: 1, thirds: 0.9, contrast: 1 }, + { tMs: 5000, sharpness: 0.8, face: 0.5, thirds: 0.5, contrast: 0.5 }, + { tMs: 9000, sharpness: 0.7, face: 0.5, thirds: 0.5, contrast: 0.5 }, + ]; + expect(rankThumbnailFrames(candidates, 3)).toEqual([1000, 5000, 9000]); + expect(rankThumbnailFrames(candidates.slice(1, 3), 3)).toEqual([1000, 1200]); + expect(rankThumbnailFrames(candidates, 0)).toEqual([]); + }); +}); diff --git a/packages/media/src/test-fixtures.ts b/packages/media/src/test-fixtures.ts new file mode 100644 index 0000000..105843f --- /dev/null +++ b/packages/media/src/test-fixtures.ts @@ -0,0 +1,135 @@ +import type { BrandKit, Edl, PlatformProfile } from "@rakazo/contracts"; +import { BrandKitSchema, EdlSchema, PlatformProfileSchema } from "@rakazo/contracts"; +import type { EdlContext } from "./edl.js"; + +/* Synthetic fixtures shared by the unit tests. No real people, places, or platforms. */ + +export function testProfile(overrides: Record = {}): PlatformProfile { + return PlatformProfileSchema.parse({ + id: "example-short-vertical", + name: "Example short vertical", + kind: "free_social", + reviewedAt: "2026-09-01", + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 500, + }, + duration: { hardMinMs: 1000, hardMaxMs: 60_000, recommended: [7000, 15_000, 30_000] }, + safeZones: { topPct: 10, bottomPct: 20, leftPct: 5, rightPct: 12 }, + captions: { recommended: true, maxLines: 2 }, + audio: {}, + text: { + titleMaxChars: 100, + captionMaxChars: 150, + hashtagLimit: 3, + linkPolicy: "bio_only", + bannedPatterns: [{ pattern: "free\\s+money", reason: "misleading claim" }], + }, + content: { thresholds: { explicit_nudity: 0.2, violence: 0.5 } }, + disclosure: { aiAlteredLabelRequired: true }, + consent: { requirePerformerConsent: false, requireAgeVerification: false }, + export: {}, + ...overrides, + }); +} + +export function testPaidProfile(): PlatformProfile { + return testProfile({ + id: "example-paid-video", + name: "Example paid video", + kind: "paid_video", + video: { + aspectRatios: ["16:9"], + width: 1920, + height: 1080, + fps: 30, + codec: "h264", + maxBitrateKbps: 12_000, + maxFileSizeMb: 4000, + }, + duration: { hardMinMs: 1000, hardMaxMs: 3_600_000, recommended: [] }, + safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 }, + text: { captionMaxChars: 2000, hashtagLimit: 10, linkPolicy: "allowed" }, + consent: { requirePerformerConsent: true, requireAgeVerification: true }, + disclosure: { aiAlteredLabelRequired: true, ageRestrictedFlag: true }, + }); +} + +export function testKit(overrides: Partial = {}): BrandKit { + return BrandKitSchema.parse({ + name: "Studio Example", + colors: { + primary: "#112233", + secondary: "#445566", + accent: "#FFAA00", + text: "#FFFFFF", + background: "#000000", + }, + fonts: [{ family: "Example Sans", artifactId: "font-1", role: "caption", license: "OFL" }], + watermark: { artifactId: "logo-1", position: "top_right", opacity: 0.6, scale: 0.15 }, + captionPresets: [ + { + id: "clean", + name: "Clean", + template: "clean", + fontFamily: "Example Sans", + sizePct: 5, + textColor: "#FFFFFF", + }, + ], + musicBeds: [{ artifactId: "music-1", title: "Bed", license: "own", commercialUse: true }], + ...overrides, + }); +} + +/** Two 16:9 clips of a 10 s asset, 4 s total, on a vertical profile. */ +export function testEdl(overrides: Record = {}): Edl { + return EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: "example-short-vertical", + width: 1080, + height: 1920, + fps: 30, + durationTargetMs: 4000, + durationHardMinMs: 1000, + durationHardMaxMs: 60_000, + }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 2000, reframe: "auto" }, + ], + }, + metadata: {}, + ...overrides, + }); +} + +export function testContext(overrides: Partial = {}): EdlContext { + return { + profile: testProfile(), + brandKit: testKit(), + assets: new Map([ + ["asset-a", { kind: "video", durationMs: 10_000, width: 1920, height: 1080 }], + ["asset-b", { kind: "video", durationMs: 8000, width: 1920, height: 1080 }], + ["photo-1", { kind: "photo", durationMs: null, width: 4000, height: 3000 }], + ]), + artifactIds: new Set(["music-1", "music-2", "vo-1"]), + music: new Map([ + ["music-1", { commercialUse: true }], + ["music-2", { commercialUse: false }], + ]), + markIds: new Set(["mark-1"]), + personIds: new Set(["person-1"]), + faceTrackIds: new Set(["track-1"]), + templateIds: new Set(["intro-minimal", "outro-simple", "lower-third-a"]), + paidOnly: [], + ...overrides, + }; +}