From a663764ee8b9db2c37ade5c7bc2966e4dd3f4266 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 17:32:52 +0000 Subject: [PATCH 01/16] feat(media): contracts and foundation for Media Studio Freeze the shared contracts every Media Studio slice builds on: zod DTOs, EDL, platform profile, brand kit, gate and render plan schemas; the media RPC surface with a scaffold router; provider interfaces for sources, inference, rendering, motion graphics and publishing; media background jobs; the Prisma models and migration; and the @rakazo/media domain package signatures. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- .env.example | 7 + apps/api/package.json | 1 + apps/api/src/media.ts | 113 ++ apps/api/src/router.ts | 2 + apps/mobile/package.json | 2 + apps/web/package.json | 1 + .../adapter-kit/src/background-jobs.test.ts | 6 + packages/adapter-kit/src/background-jobs.ts | 49 + packages/adapter-kit/src/index.ts | 1 + packages/adapter-kit/src/media.ts | 325 ++++ packages/adapter-kit/src/registry.ts | 4 + packages/adapter-kit/src/types.ts | 22 +- packages/adapters/package.json | 2 + .../adapters/src/background-job-handlers.ts | 18 + .../adapters/src/wakeup-graphile-host.test.ts | 6 + packages/adapters/src/wakeup.postgres.test.ts | 6 + packages/adapters/src/wakeup.test.ts | 6 + packages/contracts/src/index.ts | 1 + packages/contracts/src/media.ts | 1518 +++++++++++++++++ packages/contracts/src/rpc.ts | 95 ++ .../20261003172425_media_studio/migration.sql | 513 ++++++ packages/db/prisma/schema.prisma | 425 +++++ packages/media/package.json | 30 + packages/media/src/clustering.ts | 87 + packages/media/src/compile.ts | 117 ++ packages/media/src/edl.ts | 45 + packages/media/src/gate.ts | 87 + packages/media/src/geo.ts | 28 + packages/media/src/index.ts | 8 + packages/media/src/marketing.ts | 66 + packages/media/src/node.ts | 18 + packages/media/src/not-implemented.ts | 4 + packages/media/src/profiles.ts | 38 + packages/media/src/teaser.ts | 98 ++ packages/media/tsconfig.json | 8 + pnpm-lock.yaml | 202 +++ 36 files changed, 3958 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/media.ts create mode 100644 packages/adapter-kit/src/media.ts create mode 100644 packages/contracts/src/media.ts create mode 100644 packages/db/prisma/migrations/20261003172425_media_studio/migration.sql create mode 100644 packages/media/package.json create mode 100644 packages/media/src/clustering.ts create mode 100644 packages/media/src/compile.ts create mode 100644 packages/media/src/edl.ts create mode 100644 packages/media/src/gate.ts create mode 100644 packages/media/src/geo.ts create mode 100644 packages/media/src/index.ts create mode 100644 packages/media/src/marketing.ts create mode 100644 packages/media/src/node.ts create mode 100644 packages/media/src/not-implemented.ts create mode 100644 packages/media/src/profiles.ts create mode 100644 packages/media/src/teaser.ts create mode 100644 packages/media/tsconfig.json diff --git a/.env.example b/.env.example index 5d588c8..5f8d9eb 100644 --- a/.env.example +++ b/.env.example @@ -236,3 +236,10 @@ EVOO_COCKPIT_CORS_ORIGIN= # Expo: English or Simplified Chinese (en | zh-CN). Overridden by SecureStore key # rakazo.uiLocale from Account → Language. # EXPO_PUBLIC_DEFAULT_UI_LOCALE=en + +# Optional Media Studio worker (local ffmpeg + ML sidecar, compose profile "media"). +# Without it, Media Studio stays hidden and media jobs are skipped. +# MEDIA_WORKER_URL=http://127.0.0.1:8790 +# MEDIA_WORKER_TOKEN=REPLACE_WITH_RANDOM_TOKEN +# Private platform profiles that override the shipped ones. Keep outside the repo. +# MEDIA_PROFILES_DIR=~/.evoo/media-profiles diff --git a/apps/api/package.json b/apps/api/package.json index 6fbddbe..f2ef08c 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -19,6 +19,7 @@ "@rakazo/core": "workspace:*", "@rakazo/db": "workspace:*", "@rakazo/logging": "workspace:*", + "@rakazo/media": "workspace:*", "@rakazo/memory": "workspace:*", "hono": "4.13.7" }, diff --git a/apps/api/src/media.ts b/apps/api/src/media.ts new file mode 100644 index 0000000..6b4ec4a --- /dev/null +++ b/apps/api/src/media.ts @@ -0,0 +1,113 @@ +import { implement, ORPCError } from "@orpc/server"; +import type { Actor } from "@rakazo/contracts"; +import { appContract } from "@rakazo/contracts"; +import type { PrismaClient } from "@rakazo/db"; + +export interface MediaRouterDeps { + prisma: PrismaClient; + dataDir: string; +} + +// Phase A scaffold: every procedure is replaced by its real implementation. +function notImplemented(): never { + throw new ORPCError("NOT_IMPLEMENTED", { message: "Media Studio is not available yet" }); +} + +export function createMediaRouter(_deps: MediaRouterDeps) { + const os = implement(appContract.media).$context<{ actor: Actor | null; signal?: AbortSignal }>(); + const authed = os.use(async ({ context, next }) => { + if (!context.actor) throw new ORPCError("UNAUTHORIZED"); + return next({ context: { ...context, actor: context.actor } }); + }); + + return os.router({ + capabilities: authed.capabilities.handler(notImplemented), + sources: { + list: authed.sources.list.handler(notImplemented), + createFolder: authed.sources.createFolder.handler(notImplemented), + registerClient: authed.sources.registerClient.handler(notImplemented), + pushItems: authed.sources.pushItems.handler(notImplemented), + sync: authed.sources.sync.handler(notImplemented), + remove: authed.sources.remove.handler(notImplemented), + }, + assets: { + search: authed.assets.search.handler(notImplemented), + get: authed.assets.get.handler(notImplemented), + setPaidOnly: authed.assets.setPaidOnly.handler(notImplemented), + requestUpload: authed.assets.requestUpload.handler(notImplemented), + }, + uploads: { + begin: authed.uploads.begin.handler(notImplemented), + status: authed.uploads.status.handler(notImplemented), + complete: authed.uploads.complete.handler(notImplemented), + }, + collections: { + list: authed.collections.list.handler(notImplemented), + get: authed.collections.get.handler(notImplemented), + rename: authed.collections.rename.handler(notImplemented), + merge: authed.collections.merge.handler(notImplemented), + split: authed.collections.split.handler(notImplemented), + createManual: authed.collections.createManual.handler(notImplemented), + recluster: authed.collections.recluster.handler(notImplemented), + }, + people: { + list: authed.people.list.handler(notImplemented), + update: authed.people.update.handler(notImplemented), + merge: authed.people.merge.handler(notImplemented), + assignTrack: authed.people.assignTrack.handler(notImplemented), + recordConsent: authed.people.recordConsent.handler(notImplemented), + remove: authed.people.remove.handler(notImplemented), + }, + marks: { + list: authed.marks.list.handler(notImplemented), + create: authed.marks.create.handler(notImplemented), + update: authed.marks.update.handler(notImplemented), + remove: authed.marks.remove.handler(notImplemented), + }, + brandKits: { + list: authed.brandKits.list.handler(notImplemented), + upsert: authed.brandKits.upsert.handler(notImplemented), + remove: authed.brandKits.remove.handler(notImplemented), + templates: authed.brandKits.templates.handler(notImplemented), + introVariants: authed.brandKits.introVariants.handler(notImplemented), + }, + profiles: { + list: authed.profiles.list.handler(notImplemented), + get: authed.profiles.get.handler(notImplemented), + }, + projects: { + list: authed.projects.list.handler(notImplemented), + get: authed.projects.get.handler(notImplemented), + create: authed.projects.create.handler(notImplemented), + update: authed.projects.update.handler(notImplemented), + remove: authed.projects.remove.handler(notImplemented), + autoPlan: authed.projects.autoPlan.handler(notImplemented), + }, + plans: { + submit: authed.plans.submit.handler(notImplemented), + }, + renders: { + start: authed.renders.start.handler(notImplemented), + get: authed.renders.get.handler(notImplemented), + cancel: authed.renders.cancel.handler(notImplemented), + thumbnails: authed.renders.thumbnails.handler(notImplemented), + }, + marketing: { + copy: authed.marketing.copy.handler(notImplemented), + }, + publications: { + list: authed.publications.list.handler(notImplemented), + create: authed.publications.create.handler(notImplemented), + bundle: authed.publications.bundle.handler(notImplemented), + markPublished: authed.publications.markPublished.handler(notImplemented), + cancel: authed.publications.cancel.handler(notImplemented), + }, + metrics: { + mappings: authed.metrics.mappings.handler(notImplemented), + saveMapping: authed.metrics.saveMapping.handler(notImplemented), + importCsv: authed.metrics.importCsv.handler(notImplemented), + record: authed.metrics.record.handler(notImplemented), + funnel: authed.metrics.funnel.handler(notImplemented), + }, + }); +} diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 963a533..b69c2a9 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -166,6 +166,7 @@ import { renameMainConversation, } from "./main-conversation.js"; import { buildMcpUpdateMaterial } from "./mcp-material.js"; +import { createMediaRouter } from "./media.js"; import { disconnectMemoryProvider, getMemoryProviderConfig, @@ -526,6 +527,7 @@ export function createRouter(deps: RouterDeps) { }); return os.router({ + media: createMediaRouter({ prisma: deps.prisma, dataDir: deps.dataDir }), aiConsent: { status: authed.aiConsent.status.handler(({ context, input }) => aiConsentStatus(deps, context.actor, input), diff --git a/apps/mobile/package.json b/apps/mobile/package.json index d5bb46b..176c162 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -35,6 +35,7 @@ "expo-image-picker": "57.0.20", "expo-linking": "57.0.11", "expo-localization": "57.0.2", + "expo-media-library": "57.0.5", "expo-modules-core": "57.0.20", "expo-notifications": "57.0.21", "expo-router": "57.0.24", @@ -44,6 +45,7 @@ "expo-status-bar": "57.0.1", "expo-symbols": "57.0.3", "expo-updates": "57.0.24", + "expo-video": "57.0.5", "react": "19.2.3", "react-dom": "19.2.3", "react-native": "0.86.3", diff --git a/apps/web/package.json b/apps/web/package.json index 1f121dc..a9326b8 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -24,6 +24,7 @@ "@rakazo/core": "workspace:*", "@rakazo/ui-tokens": "workspace:*", "@rakazo/ui-web": "workspace:*", + "@tanstack/react-virtual": "3.14.13", "better-auth": "1.7.4", "lucide-react": "1.44.0", "react": "19.2.3", diff --git a/packages/adapter-kit/src/background-jobs.test.ts b/packages/adapter-kit/src/background-jobs.test.ts index 13880d7..df0f258 100644 --- a/packages/adapter-kit/src/background-jobs.test.ts +++ b/packages/adapter-kit/src/background-jobs.test.ts @@ -22,6 +22,12 @@ function handlers(): BackgroundJobHandlers { "messaging.deliver": vi.fn(async () => undefined), "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), + "media.sync": vi.fn(async () => undefined), + "media.proxy": vi.fn(async () => undefined), + "media.analyze": vi.fn(async () => undefined), + "media.cluster": vi.fn(async () => undefined), + "media.render": vi.fn(async () => undefined), + "media.publish": vi.fn(async () => undefined), }; } diff --git a/packages/adapter-kit/src/background-jobs.ts b/packages/adapter-kit/src/background-jobs.ts index 45cad56..3f09390 100644 --- a/packages/adapter-kit/src/background-jobs.ts +++ b/packages/adapter-kit/src/background-jobs.ts @@ -1,3 +1,5 @@ +import type { MediaAnalysisKind } from "@rakazo/contracts"; +import { MediaAnalysisKindSchema } from "@rakazo/contracts"; import { z } from "zod"; import type { BackgroundJob, @@ -23,6 +25,15 @@ const payloadSchemas = { "messaging.deliver": z.object({ runId: z.string().min(1).optional() }), "cloud_agent.poll": z.object({ agentId: z.string().min(1) }), "profile.consolidate": z.object({ userId: z.string().min(1) }), + "media.sync": z.object({ sourceId: z.string().min(1) }), + "media.proxy": z.object({ assetId: z.string().min(1) }), + "media.analyze": z.object({ + assetId: z.string().min(1), + kinds: z.array(MediaAnalysisKindSchema).optional(), + }), + "media.cluster": z.object({ spaceId: z.string().min(1) }), + "media.render": z.object({ renderId: z.string().min(1) }), + "media.publish": z.object({ publicationId: z.string().min(1) }), } satisfies { [Name in BackgroundJobName]: z.ZodType }; export function parseBackgroundJob(name: string, payload: unknown): BackgroundJob { @@ -160,3 +171,41 @@ export function profileConsolidateJob(userId: string): BackgroundJob { replaceKey: profileConsolidateJobKey(userId), }; } + +export function mediaSyncJob(sourceId: string): BackgroundJob { + return { name: "media.sync", payload: { sourceId }, replaceKey: `media.sync:${sourceId}` }; +} + +export function mediaProxyJob(assetId: string): BackgroundJob { + return { name: "media.proxy", payload: { assetId }, replaceKey: `media.proxy:${assetId}` }; +} + +export function mediaAnalyzeJob(assetId: string, kinds?: MediaAnalysisKind[]): BackgroundJob { + return { + name: "media.analyze", + payload: kinds ? { assetId, kinds } : { assetId }, + replaceKey: `media.analyze:${assetId}`, + }; +} + +export function mediaClusterJob(spaceId: string, availableAt?: Date): BackgroundJob { + return { + name: "media.cluster", + payload: { spaceId }, + replaceKey: `media.cluster:${spaceId}`, + ...(availableAt ? { availableAt } : {}), + }; +} + +export function mediaRenderJob(renderId: string): BackgroundJob { + return { name: "media.render", payload: { renderId }, replaceKey: `media.render:${renderId}` }; +} + +export function mediaPublishJob(publicationId: string, availableAt?: Date): BackgroundJob { + return { + name: "media.publish", + payload: { publicationId }, + replaceKey: `media.publish:${publicationId}`, + ...(availableAt ? { availableAt } : {}), + }; +} diff --git a/packages/adapter-kit/src/index.ts b/packages/adapter-kit/src/index.ts index 43f7898..e39a95c 100644 --- a/packages/adapter-kit/src/index.ts +++ b/packages/adapter-kit/src/index.ts @@ -1,5 +1,6 @@ export * from "./background-jobs.js"; export * from "./cloud-agents.js"; export * from "./interfaces.js"; +export * from "./media.js"; export * from "./registry.js"; export * from "./types.js"; diff --git a/packages/adapter-kit/src/media.ts b/packages/adapter-kit/src/media.ts new file mode 100644 index 0000000..1674304 --- /dev/null +++ b/packages/adapter-kit/src/media.ts @@ -0,0 +1,325 @@ +import type { + AnonymizeMode, + BoxKeyframe, + BrandKit, + ContentCategory, + FfprobeReport, + MediaAssetKind, + MediaCapabilities, + MediaInferenceOperation, + MediaMarkPrompt, + MediaSourceKind, + MotionTemplateInfo, + MotionTemplateRef, + NormalizedBox, + NormalizedPoint, + PlatformProfile, + RenderPlan, +} from "@rakazo/contracts"; + +/* + * Media Studio provider boundaries. Every implementation (local sidecar, + * emulator, optional cloud adapters) satisfies the same contracts, which + * media-conformance tests exercise offline. + * + * Paths are absolute host paths inside the configured media root. Providers + * must reject any path outside that root. + */ + +// --------------------------------------------------------------------------- +// Sources +// --------------------------------------------------------------------------- + +export interface MediaSourceItem { + externalId: string; + kind: MediaAssetKind; + capturedAt: Date | null; + durationMs: number | null; + width: number | null; + height: number | null; + fps: number | null; + gpsLat: number | null; + gpsLon: number | null; + deviceModel: string | null; + albumNames: string[]; + isFavorite: boolean; + fileName: string | null; + sizeBytes: number | null; + /** Cheap change detector (mtime + size or provider revision); not a content hash. */ + revision: string; +} + +export interface MediaSourceListRequest { + config: Record; + cursor: string | null; + limit: number; + signal?: AbortSignal; +} + +export interface MediaSourceListResult { + items: MediaSourceItem[]; + nextCursor: string | null; +} + +/** Server-side sources the worker can enumerate and read directly. */ +export interface MediaSourceProvider { + readonly kind: Extract; + list(request: MediaSourceListRequest): Promise; + /** Absolute readable path of the original file. Throws when it escaped the source root. */ + resolve(config: Record, externalId: string): Promise; +} + +// --------------------------------------------------------------------------- +// Inference +// --------------------------------------------------------------------------- + +export class UnsupportedMediaOperationError extends Error { + constructor( + readonly operation: MediaInferenceOperation, + readonly reason: string, + ) { + super(`${operation} is not available: ${reason}`); + this.name = "UnsupportedMediaOperationError"; + } +} + +export interface MediaTimeSpan { + startMs: number; + endMs: number; +} + +export interface DetectScenesResult { + shots: Array; + scenes: MediaTimeSpan[]; +} + +export interface TranscriptWord { + startMs: number; + endMs: number; + text: string; + confidence: number; +} + +export interface TranscribeResult { + language: string | null; + words: TranscriptWord[]; +} + +export interface FaceObservation { + box: NormalizedBox; + confidence: number; + /** Five-point landmarks: eyes, nose, mouth corners. */ + landmarks?: NormalizedPoint[]; +} + +export interface DetectFacesResult { + frames: Array<{ tMs: number; faces: FaceObservation[] }>; +} + +export interface FaceTrackResult { + trackId: string; + keyframes: Array; + /** Identity embedding for local person clustering. Never leaves the host. */ + embedding: number[] | null; +} + +export interface TrackFacesResult { + sampleFps: number; + tracks: FaceTrackResult[]; +} + +export interface FaceLandmarksResult { + frames: Array<{ + tMs: number; + faces: Array<{ box: NormalizedBox; landmarks: NormalizedPoint[] }>; + }>; +} + +export interface EmbedFramesResult { + model: string; + dim: number; + vectors: number[][]; +} + +export interface ClassifyContentResult { + model: string; + frames: Array<{ tMs: number; scores: Partial> }>; +} + +export interface SegmentPromptedResult { + /** Directory of per-frame PNG masks named by timestamp in ms. */ + maskDir: string; + keyframes: Array; + score: number; +} + +export interface InpaintQuality { + /** 1 = no flicker inside the mask region between neighbouring frames. */ + temporalStability: number; + /** 1 = no visible seams at the mask edge. */ + edgeQuality: number; + score: number; +} + +export interface InpaintVideoResult { + outPath: string; + quality: InpaintQuality; +} + +export interface AudioFeaturesResult { + integratedLufs: number | null; + truePeakDb: number | null; + tempoBpm: number | null; + beatsMs: number[]; + energy: Array<{ tMs: number; value: number }>; + silences: MediaTimeSpan[]; +} + +export interface SaliencyResult { + points: Array<{ tMs: number; cx: number; cy: number; strength: number }>; +} + +export interface RedactionInstruction { + keyframes: BoxKeyframe[]; + mode: Exclude | "cover"; + paddingPct: number; + featherPct: number; + /** Solid and cover fill. */ + color?: string; + /** Landmarks per keyframe for partial modes. */ + landmarks?: Array<{ tMs: number; points: NormalizedPoint[] }>; + /** Mask directory for region marks instead of boxes. */ + maskDir?: string; +} + +interface MediaPathRequest { + path: string; + signal?: AbortSignal; +} + +/** + * Local-first ML operations. Callers check `capabilities()` and fall back + * (blur, cover, reframe) when an operation reports unavailable; providers + * throw UnsupportedMediaOperationError instead of guessing. + */ +export interface MediaInferenceProvider { + readonly id: string; + capabilities(signal?: AbortSignal): Promise; + detectScenes(request: MediaPathRequest): Promise; + transcribe(request: MediaPathRequest & { language?: string }): Promise; + detectFaces( + request: MediaPathRequest & { sampleFps: number; timesMs?: number[] }, + ): Promise; + trackFaces(request: MediaPathRequest & { sampleFps: number }): Promise; + faceLandmarks(request: MediaPathRequest & { timesMs: number[] }): Promise; + embedFrames(request: MediaPathRequest & { timesMs: number[] }): Promise; + classifyContent( + request: MediaPathRequest & { sampleFps: number; extraTimesMs?: number[] }, + ): Promise; + segmentPrompted( + request: MediaPathRequest & { prompt: MediaMarkPrompt; propagate: boolean; outDir: string }, + ): Promise; + inpaintVideo( + request: MediaPathRequest & { maskDir: string; outPath: string }, + ): Promise; + /** + * Apply face and region redactions frame-accurately and write a near-lossless + * intermediate. Fails closed: a frame it cannot process is an error, not a pass. + */ + redact( + request: MediaPathRequest & { redactions: RedactionInstruction[]; outPath: string }, + ): Promise<{ outPath: string; framesProcessed: number }>; + /** + * Replace a tracked face with a generated, non-existent identity derived from + * `seed`. Deliberately takes no target-face image. + */ + synthesizeIdentity( + request: MediaPathRequest & { track: BoxKeyframe[]; seed: number; outPath: string }, + ): Promise<{ outPath: string }>; + audioFeatures(request: MediaPathRequest): Promise; + saliency(request: MediaPathRequest & { timesMs: number[] }): Promise; +} + +// --------------------------------------------------------------------------- +// Rendering +// --------------------------------------------------------------------------- + +export interface MediaRenderProgress { + stepId: string; + /** Overall progress 0..1. */ + progress: number; +} + +export interface MediaRenderer { + readonly id: string; + /** Hardware encoders usable on this host, e.g. h264_vaapi, h264_videotoolbox. */ + hardwareEncoders(): Promise; + /** + * Run every ffmpeg step of the plan. Motion, inpaint, and synthetic identity + * steps must already be satisfied (their outPath exists) or the renderer throws. + */ + render( + plan: RenderPlan, + options: { signal?: AbortSignal; onProgress?: (progress: MediaRenderProgress) => void }, + ): Promise; + probe( + path: string, + options?: { measureLoudness?: boolean; signal?: AbortSignal }, + ): Promise; + /** Extract a still frame as PNG (EXIF-free). */ + extractFrame(input: { + path: string; + tMs: number; + outPath: string; + width?: number; + signal?: AbortSignal; + }): Promise; +} + +export interface MotionRenderRequest { + template: MotionTemplateRef; + brandKit: BrandKit; + /** Resolved font and logo files from the brand kit artifacts. */ + assets: { fonts: Array<{ family: string; path: string }>; logoPaths: string[] }; + width: number; + height: number; + fps: number; + outPath: string; + signal?: AbortSignal; +} + +export interface MotionGraphicsRenderer { + readonly id: string; + templates(): MotionTemplateInfo[]; + /** Render an overlay clip with alpha (MOV, qtrle or prores 4444). */ + render(request: MotionRenderRequest): Promise<{ path: string; hasAlpha: boolean }>; +} + +// --------------------------------------------------------------------------- +// Publishing +// --------------------------------------------------------------------------- + +export interface MediaPublishRequest { + bundleDir: string; + profile: PlatformProfile; + publicationId: string; + title: string | null; + caption: string | null; + tags: string[]; + signal?: AbortSignal; +} + +export interface MediaPublishResult { + status: "published" | "bundled"; + externalUrl: string | null; +} + +export interface MediaPublisher { + readonly id: string; + publish(request: MediaPublishRequest): Promise; +} + +export interface LinkShortener { + readonly id: string; + shorten(url: string, signal?: AbortSignal): Promise; +} diff --git a/packages/adapter-kit/src/registry.ts b/packages/adapter-kit/src/registry.ts index 19c0822..086efa2 100644 --- a/packages/adapter-kit/src/registry.ts +++ b/packages/adapter-kit/src/registry.ts @@ -36,4 +36,8 @@ export const slots = { web: "web", browser: "browser", cloudAgent: "cloud-agent", + mediaInference: "media-inference", + mediaRenderer: "media-renderer", + motionGraphics: "motion-graphics", + mediaPublisher: "media-publisher", } as const; diff --git a/packages/adapter-kit/src/types.ts b/packages/adapter-kit/src/types.ts index d77cb5e..7bf3883 100644 --- a/packages/adapter-kit/src/types.ts +++ b/packages/adapter-kit/src/types.ts @@ -1,4 +1,9 @@ -import type { ConnectionCatalogItem, ModelTier, SandboxKind } from "@rakazo/contracts"; +import type { + ConnectionCatalogItem, + MediaAnalysisKind, + ModelTier, + SandboxKind, +} from "@rakazo/contracts"; export interface AdapterContext { operationId: string; @@ -524,9 +529,22 @@ export interface BackgroundJobPayloads { "cloud_agent.poll": { agentId: string }; /** Learn durable user facts from conversations since the profile's last consolidation. */ "profile.consolidate": { userId: string }; + /** Enumerate a server-side media source incrementally (metadata only). */ + "media.sync": { sourceId: string }; + /** Hash the original and build proxy, thumbnails, and keyframes. */ + "media.proxy": { assetId: string }; + /** Run sidecar analyses; omitted kinds means every analysis not yet at the current version. */ + "media.analyze": { assetId: string; kinds?: MediaAnalysisKind[] }; + /** Rebuild sessions, themes, and person suggestions for a space. */ + "media.cluster": { spaceId: string }; + /** Render, probe, and gate one MediaRender. */ + "media.render": { renderId: string }; + /** Build the export bundle and publish a scheduled MediaPublication. */ + "media.publish": { publicationId: string }; } export type BackgroundJobName = keyof BackgroundJobPayloads; +export type MediaJobName = Extract; export type BackgroundJob = { [Name in BackgroundJobName]: { @@ -541,6 +559,8 @@ export type BackgroundJobHandlers = { [Name in BackgroundJobName]: (payload: BackgroundJobPayloads[Name]) => Promise; }; +export type MediaJobHandlers = Pick; + export interface SecretRecord { id: string; ciphertext: string; diff --git a/packages/adapters/package.json b/packages/adapters/package.json index c9b7855..d2e461d 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -29,6 +29,8 @@ "@rakazo/core": "workspace:*", "@rakazo/db": "workspace:*", "@rakazo/logging": "workspace:*", + "@rakazo/media": "workspace:*", + "@resvg/resvg-js": "2.6.2", "chat": "4.40.0", "chat-adapter-lark": "0.3.1", "chat-adapter-sendblue": "0.2.0", diff --git a/packages/adapters/src/background-job-handlers.ts b/packages/adapters/src/background-job-handlers.ts index af869ac..949c898 100644 --- a/packages/adapters/src/background-job-handlers.ts +++ b/packages/adapters/src/background-job-handlers.ts @@ -3,6 +3,7 @@ import type { AgentRuntime, BackgroundJobHandlers, JobPublisher, + MediaJobHandlers, MessagingSurface, SandboxProvider, } from "@rakazo/adapter-kit"; @@ -36,6 +37,8 @@ export function createBackgroundJobHandlers(deps: { deploymentModelKey?: string; messaging?: MessagingSurface; cloudAgent?: CloudAgentConnection | null; + /** Media Studio job handlers; absent when media is not configured for this worker. */ + media?: MediaJobHandlers; }): BackgroundJobHandlers { const deliverMessaging = async (runId?: string) => { if (!deps.messaging) return; @@ -53,6 +56,7 @@ export function createBackgroundJobHandlers(deps: { }; return { + ...(deps.media ?? mediaDisabledHandlers()), "run.continue": async (payload) => { await deps.executor.continueRun(payload.runId, deps.workerId); // Automatic messaging mirror: once the run's bot messages are durable, @@ -126,3 +130,17 @@ export function createBackgroundJobHandlers(deps: { }, }; } + +function mediaDisabledHandlers(): MediaJobHandlers { + const skip = async () => { + getLogger().warn("media job skipped: Media Studio is not configured on this worker"); + }; + return { + "media.sync": skip, + "media.proxy": skip, + "media.analyze": skip, + "media.cluster": skip, + "media.render": skip, + "media.publish": skip, + }; +} diff --git a/packages/adapters/src/wakeup-graphile-host.test.ts b/packages/adapters/src/wakeup-graphile-host.test.ts index b6cc385..4aecbd2 100644 --- a/packages/adapters/src/wakeup-graphile-host.test.ts +++ b/packages/adapters/src/wakeup-graphile-host.test.ts @@ -24,6 +24,12 @@ function handlers(): BackgroundJobHandlers { "messaging.deliver": vi.fn(async () => undefined), "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), + "media.sync": vi.fn(async () => undefined), + "media.proxy": vi.fn(async () => undefined), + "media.analyze": vi.fn(async () => undefined), + "media.cluster": vi.fn(async () => undefined), + "media.render": vi.fn(async () => undefined), + "media.publish": vi.fn(async () => undefined), }; } diff --git a/packages/adapters/src/wakeup.postgres.test.ts b/packages/adapters/src/wakeup.postgres.test.ts index b164554..be759c7 100644 --- a/packages/adapters/src/wakeup.postgres.test.ts +++ b/packages/adapters/src/wakeup.postgres.test.ts @@ -29,6 +29,12 @@ function handlers(overrides: Partial = {}): BackgroundJob "messaging.deliver": vi.fn(async () => undefined), "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), + "media.sync": vi.fn(async () => undefined), + "media.proxy": vi.fn(async () => undefined), + "media.analyze": vi.fn(async () => undefined), + "media.cluster": vi.fn(async () => undefined), + "media.render": vi.fn(async () => undefined), + "media.publish": vi.fn(async () => undefined), ...overrides, }; } diff --git a/packages/adapters/src/wakeup.test.ts b/packages/adapters/src/wakeup.test.ts index 3856671..b5eb13c 100644 --- a/packages/adapters/src/wakeup.test.ts +++ b/packages/adapters/src/wakeup.test.ts @@ -14,6 +14,12 @@ function handlers(): BackgroundJobHandlers { "messaging.deliver": vi.fn(async () => undefined), "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), + "media.sync": vi.fn(async () => undefined), + "media.proxy": vi.fn(async () => undefined), + "media.analyze": vi.fn(async () => undefined), + "media.cluster": vi.fn(async () => undefined), + "media.render": vi.fn(async () => undefined), + "media.publish": vi.fn(async () => undefined), }; } diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 3865e1b..ba2c70b 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -9,6 +9,7 @@ export * from "./inbox.js"; export * from "./integration-settings.js"; export * from "./local-settings.js"; export * from "./mcp.js"; +export * from "./media.js"; export * from "./model-tiers.js"; export * from "./openai-compatible-ui.js"; export * from "./project-goals.js"; diff --git a/packages/contracts/src/media.ts b/packages/contracts/src/media.ts new file mode 100644 index 0000000..428fa44 --- /dev/null +++ b/packages/contracts/src/media.ts @@ -0,0 +1,1518 @@ +import * as z from "zod"; +import { Id, IsoDate } from "./ids.js"; + +/* + * Media Studio contracts: the single source of truth for media DTOs, the edit + * decision list (EDL), platform profiles, brand kits, gate reports, render + * plans, and the media_* agent tool inputs. + */ + +// --------------------------------------------------------------------------- +// Enums +// --------------------------------------------------------------------------- + +export const MediaSourceKindSchema = z.enum(["apple_photos", "ios_library", "folder", "emulator"]); +export type MediaSourceKind = z.infer; + +/** Sources whose files live on a client device and are pushed to the server. */ +export const CLIENT_PUSHED_MEDIA_SOURCES = ["apple_photos", "ios_library"] as const; + +export const MediaSourceStatusSchema = z.enum(["idle", "syncing", "error", "unavailable"]); +export type MediaSourceStatus = z.infer; + +export const MediaAssetKindSchema = z.enum(["video", "photo", "live_photo"]); +export type MediaAssetKind = z.infer; + +/** Where the original bytes are: referenced in place, uploaded to DATA_DIR, or still on a device. */ +export const MediaAssetStorageSchema = z.enum(["referenced", "uploaded", "remote"]); +export type MediaAssetStorage = z.infer; + +export const MediaIngestStateSchema = z.enum([ + "indexed", + "awaiting_upload", + "proxying", + "proxied", + "analyzing", + "analyzed", + "failed", +]); +export type MediaIngestState = z.infer; + +export const MediaSegmentKindSchema = z.enum(["shot", "scene"]); +export type MediaSegmentKind = z.infer; + +export const MediaAnalysisKindSchema = z.enum([ + "scenes", + "transcript", + "faces", + "embeddings", + "classifier", + "audio", + "saliency", +]); +export type MediaAnalysisKind = z.infer; + +export const MediaCollectionKindSchema = z.enum(["session", "theme", "manual"]); +export type MediaCollectionKind = z.infer; + +export const MediaLabelSourceSchema = z.enum(["rule", "model", "user"]); +export type MediaLabelSource = z.infer; + +export const AnonymizeModeSchema = z.enum([ + "none", + "blur", + "pixelate", + "solid", + "sticker", + "partial_eyes", + "partial_upper", + "partial_lower", + "reframe_out", + "synthetic_identity", +]); +export type AnonymizeMode = z.infer; + +export const AgeVerificationMethodSchema = z.enum([ + "platform_verified", + "document_on_file", + "none", +]); +export type AgeVerificationMethod = z.infer; + +export const MediaMarkKindSchema = z.enum([ + "tattoo", + "birthmark", + "scar", + "jewelry", + "background_detail", + "license_plate", + "custom", +]); +export type MediaMarkKind = z.infer; + +export const MarkTreatmentSchema = z.enum(["inpaint", "blur", "cover", "reframe_out", "none"]); +export type MarkTreatment = z.infer; + +export const MediaProjectGoalSchema = z.enum([ + "teaser", + "full_release", + "trailer", + "highlight", + "story", + "custom", +]); +export type MediaProjectGoal = z.infer; + +export const MediaProjectStatusSchema = z.enum([ + "draft", + "planning", + "rendering", + "review", + "ready", + "archived", +]); +export type MediaProjectStatus = z.infer; + +export const MediaRenderStatusSchema = z.enum([ + "queued", + "rendering", + "gating", + "blocked", + "done", + "failed", + "canceled", +]); +export type MediaRenderStatus = z.infer; + +export const MediaRenderQualitySchema = z.enum(["draft", "final"]); +export type MediaRenderQuality = z.infer; + +export const MediaPublicationStatusSchema = z.enum([ + "draft", + "scheduled", + "bundled", + "published", + "failed", + "canceled", +]); +export type MediaPublicationStatus = z.infer; + +export const MediaMetricSourceSchema = z.enum(["csv_import", "manual", "connector"]); +export type MediaMetricSource = z.infer; + +export const PlatformProfileKindSchema = z.enum([ + "free_social", + "paid_video", + "own_site", + "messenger", +]); +export type PlatformProfileKind = z.infer; + +export const ContentCategorySchema = z.enum([ + "explicit_nudity", + "suggestive", + "violence", + "minors_risk", + "self_harm", + "drugs", + "weapons", +]); +export type ContentCategory = z.infer; +export const CONTENT_CATEGORIES = ContentCategorySchema.options; + +export const LinkPolicySchema = z.enum(["allowed", "bio_only", "forbidden"]); +export type LinkPolicy = z.infer; + +export const MusicPolicySchema = z.enum(["any", "licensed_only", "platform_library_only"]); +export type MusicPolicy = z.infer; + +export const TransitionKindSchema = z.enum(["cut", "crossfade", "dip", "whip", "match_cut"]); +export type TransitionKind = z.infer; + +export const CaptionModeSchema = z.enum(["burned", "sidecar", "both", "none"]); +export type CaptionMode = z.infer; + +// --------------------------------------------------------------------------- +// Primitives +// --------------------------------------------------------------------------- + +/** Milliseconds on a media timeline. */ +export const Ms = z.number().int().min(0); +/** A normalized coordinate in [0, 1] relative to frame width or height. */ +export const Unit = z.number().min(0).max(1); +/** Identifiers that may reach a filtergraph label or file name. */ +export const SafeId = z + .string() + .min(1) + .max(64) + .regex(/^[a-z0-9_-]+$/); + +export const NormalizedBoxSchema = z.object({ x: Unit, y: Unit, w: Unit, h: Unit }); +export type NormalizedBox = z.infer; + +export const NormalizedPointSchema = z.object({ x: Unit, y: Unit }); +export type NormalizedPoint = z.infer; + +/** Hex color for brand kits and render output. Never a UI token. */ +export const BrandColor = z.string().regex(/^#[0-9a-fA-F]{6}$/); + +export const BoxKeyframeSchema = z.object({ tMs: Ms, box: NormalizedBoxSchema }); +export type BoxKeyframe = z.infer; + +// --------------------------------------------------------------------------- +// Platform profiles +// --------------------------------------------------------------------------- + +export const AspectRatioSchema = z.enum(["9:16", "4:5", "1:1", "16:9", "4:3", "21:9"]); +export type AspectRatio = z.infer; + +export const PlatformProfileSchema = z + .object({ + id: SafeId, + name: z.string().min(1).max(80), + kind: PlatformProfileKindSchema, + sourceUrls: z.array(z.string().url()).default([]), + reviewedAt: z.string().date(), + /** True when at least one value could not be verified against official documentation. */ + unverified: z.boolean().default(false), + video: z.object({ + aspectRatios: z.array(AspectRatioSchema).min(1), + width: z.number().int().min(16).max(7680), + height: z.number().int().min(16).max(7680), + fps: z.number().min(1).max(120), + codec: z.enum(["h264", "hevc"]), + maxBitrateKbps: z.number().int().positive(), + maxFileSizeMb: z.number().positive(), + }), + duration: z.object({ + hardMinMs: Ms, + hardMaxMs: Ms, + recommended: z.array(Ms).default([]), + hookWindowMs: Ms.default(2000), + }), + safeZones: z.object({ + topPct: z.number().min(0).max(50), + bottomPct: z.number().min(0).max(50), + leftPct: z.number().min(0).max(50), + rightPct: z.number().min(0).max(50), + }), + captions: z.object({ + recommended: z.boolean(), + required: z.boolean().default(false), + maxLines: z.number().int().min(1).max(4).default(2), + }), + audio: z.object({ + targetLufs: z.number().min(-30).max(-5).default(-14), + truePeakDb: z.number().min(-9).max(0).default(-1), + music: MusicPolicySchema.default("licensed_only"), + }), + text: z.object({ + titleMaxChars: z.number().int().positive().optional(), + captionMaxChars: z.number().int().positive(), + hashtagLimit: z.number().int().min(0), + linkPolicy: LinkPolicySchema, + bannedPatterns: z + .array(z.object({ pattern: z.string().min(1).max(200), reason: z.string().min(1) })) + .default([]), + }), + content: z.object({ + /** Maximum allowed classifier score per category, 0..1. Missing categories are unrestricted. */ + thresholds: z.partialRecord(ContentCategorySchema, z.number().min(0).max(1)), + sampleFps: z.number().min(0.5).max(10).default(2), + blockOn: z.enum(["any_frame", "ratio"]).default("any_frame"), + /** With blockOn "ratio": fraction of sampled frames above threshold that blocks. */ + blockRatio: z.number().min(0).max(1).optional(), + }), + disclosure: z.object({ + aiAlteredLabelRequired: z.boolean(), + paidPartnershipLabel: z.boolean().default(false), + ageRestrictedFlag: z.boolean().default(false), + }), + consent: z.object({ + requirePerformerConsent: z.boolean(), + requireAgeVerification: z.boolean(), + }), + export: z.object({ + poster: z.boolean().default(true), + subtitles: z.boolean().default(true), + captionText: z.boolean().default(true), + uploadChecklist: z.boolean().default(true), + previewClipMs: Ms.optional(), + }), + }) + .superRefine((profile, ctx) => { + if (profile.duration.hardMinMs > profile.duration.hardMaxMs) { + ctx.addIssue({ code: "custom", path: ["duration"], message: "hardMinMs > hardMaxMs" }); + } + }); +export type PlatformProfile = z.infer; + +/** Profile summary as the API exposes it. */ +export const PlatformProfileSummarySchema = z.object({ + id: SafeId, + name: z.string(), + kind: PlatformProfileKindSchema, + origin: z.enum(["builtin", "private"]), + unverified: z.boolean(), + stale: z.boolean(), + aspectRatios: z.array(AspectRatioSchema), + hardMinMs: Ms, + hardMaxMs: Ms, + recommendedMs: z.array(Ms), +}); +export type PlatformProfileSummary = z.infer; + +// --------------------------------------------------------------------------- +// Brand kits +// --------------------------------------------------------------------------- + +export const WatermarkSchema = z.object({ + artifactId: Id.optional(), + position: z.enum(["top_left", "top_right", "bottom_left", "bottom_right", "center"]), + opacity: z.number().min(0).max(1), + /** Width relative to frame width. */ + scale: z.number().min(0.02).max(0.5), +}); +export type Watermark = z.infer; + +export const CaptionStylePresetSchema = z.object({ + id: SafeId, + name: z.string().min(1).max(40), + template: z.enum(["clean", "karaoke", "bold_social"]), + fontFamily: z.string().min(1).max(80), + sizePct: z.number().min(2).max(12), + textColor: BrandColor, + highlightColor: BrandColor.optional(), + backgroundColor: BrandColor.optional(), + uppercase: z.boolean().default(false), +}); +export type CaptionStylePreset = z.infer; + +export const MotionTemplateRefSchema = z.object({ + templateId: SafeId, + params: z.record(z.string(), z.union([z.string().max(200), z.number(), z.boolean()])).default({}), + durationMs: Ms, +}); +export type MotionTemplateRef = z.infer; + +export const BrandKitSchema = z.object({ + name: z.string().min(1).max(60), + colors: z.object({ + primary: BrandColor, + secondary: BrandColor, + accent: BrandColor, + text: BrandColor, + background: BrandColor, + }), + fonts: z + .array( + z.object({ + family: z.string().min(1).max(80), + artifactId: Id, + role: z.enum(["heading", "body", "caption"]), + license: z.string().max(200).optional(), + }), + ) + .default([]), + logoArtifactIds: z.array(Id).max(4).default([]), + lutArtifactId: Id.optional(), + watermark: WatermarkSchema.optional(), + captionPresets: z.array(CaptionStylePresetSchema).default([]), + musicBeds: z + .array( + z.object({ + artifactId: Id, + title: z.string().min(1).max(80), + license: z.string().min(1).max(200), + commercialUse: z.boolean(), + }), + ) + .default([]), + voicePreset: z.object({ provider: z.string(), voiceId: z.string() }).optional(), + intro: MotionTemplateRefSchema.optional(), + outro: MotionTemplateRefSchema.optional(), + endCard: MotionTemplateRefSchema.optional(), + tone: z.string().max(200).optional(), + audience: z.string().max(200).optional(), +}); +export type BrandKit = z.infer; + +// --------------------------------------------------------------------------- +// Edit decision list +// --------------------------------------------------------------------------- + +export const ReframeKeyframeSchema = z.object({ + tMs: Ms, + /** Crop center, normalized to source frame. */ + cx: Unit, + cy: Unit, + /** 1 = tightest crop that still fills the output aspect; > 1 zooms in. */ + scale: z.number().min(1).max(4), +}); +export type ReframeKeyframe = z.infer; + +export const ColorAdjustSchema = z.object({ + exposure: z.number().min(-2).max(2).default(0), + temperature: z.number().min(-1).max(1).default(0), + saturation: z.number().min(0).max(2).default(1), + contrast: z.number().min(0.5).max(2).default(1), + useBrandLut: z.boolean().default(false), +}); +export type ColorAdjust = z.infer; + +export const EdlVideoClipSchema = z.object({ + id: SafeId, + assetId: Id, + inMs: Ms, + outMs: Ms, + /** Position on the output timeline. */ + atMs: Ms, + speed: z.number().min(0.25).max(4).default(1), + reframe: z.union([z.literal("auto"), z.literal("none"), z.array(ReframeKeyframeSchema).min(1)]), + color: ColorAdjustSchema.optional(), + stabilize: z.boolean().default(false), + muted: z.boolean().default(false), + segmentId: Id.optional(), +}); +export type EdlVideoClip = z.infer; + +export const EdlTransitionSchema = z.object({ + fromClipId: SafeId, + toClipId: SafeId, + kind: TransitionKindSchema, + durationMs: Ms.max(2000), +}); +export type EdlTransition = z.infer; + +export const EdlAudioClipSchema = z.object({ + id: SafeId, + kind: z.enum(["music", "voiceover", "sfx"]), + artifactId: Id, + inMs: Ms, + outMs: Ms, + atMs: Ms, + gainDb: z.number().min(-40).max(12).default(0), + fadeInMs: Ms.default(0), + fadeOutMs: Ms.default(0), + /** Lower this clip under speech from video clips. */ + duckUnderSpeech: z.boolean().default(false), + /** Snap cuts to this clip's beat grid. */ + beatGridMs: z.array(Ms).optional(), +}); +export type EdlAudioClip = z.infer; + +export const EdlCaptionsSchema = z.object({ + mode: CaptionModeSchema, + presetId: SafeId.optional(), + maxCharsPerLine: z.number().int().min(10).max(60).default(32), + wordHighlight: z.boolean().default(false), + language: z.string().min(2).max(10).optional(), + /** Vertical anchor; the compiler keeps the block inside the profile safe zone. */ + position: z.enum(["lower", "middle", "upper"]).default("lower"), +}); +export type EdlCaptions = z.infer; + +export const EdlOverlaySchema = z.object({ + id: SafeId, + kind: z.enum([ + "text", + "lower_third", + "logo", + "watermark", + "cta_end_card", + "progress_bar", + "motion", + ]), + atMs: Ms, + durationMs: Ms, + /** Plain text only. The compiler escapes it; it never reaches a shell. */ + text: z.string().max(200).optional(), + box: NormalizedBoxSchema.optional(), + motion: MotionTemplateRefSchema.optional(), +}); +export type EdlOverlay = z.infer; + +export const FaceRedactionSchema = z.object({ + id: SafeId, + target: z.union([ + z.object({ trackId: Id }), + z.object({ personId: Id }), + z.object({ unassignedFaces: z.literal(true) }), + ]), + mode: AnonymizeModeSchema.exclude(["none"]), + paddingPct: z.number().min(0).max(100).default(20), + featherPct: z.number().min(0).max(50).default(10), + minSizePct: z.number().min(0).max(50).default(2), +}); +export type FaceRedaction = z.infer; + +export const RegionRedactionSchema = z.object({ + id: SafeId, + clipId: SafeId, + keyframes: z.array(BoxKeyframeSchema).min(1), + treatment: MarkTreatmentSchema.exclude(["none", "inpaint"]), +}); +export type RegionRedaction = z.infer; + +export const VoiceDisguiseSchema = z.object({ + method: z.enum(["pitch", "formant"]), + semitones: z.number().min(-12).max(12), +}); +export type VoiceDisguise = z.infer; + +export const AiAlterationSchema = z.enum([ + "synthetic_identity", + "inpaint", + "voice_disguise", + "generated_voiceover", +]); +export type AiAlteration = z.infer; + +export const EDL_VERSION = 1; + +export const EdlSchema = z.object({ + edlVersion: z.literal(EDL_VERSION), + output: z.object({ + profileId: SafeId, + width: z.number().int().min(16).max(7680), + height: z.number().int().min(16).max(7680), + fps: z.number().min(1).max(120), + durationTargetMs: Ms, + durationHardMinMs: Ms, + durationHardMaxMs: Ms, + }), + tracks: z.object({ + video: z.array(EdlVideoClipSchema).min(1).max(400), + audio: z.array(EdlAudioClipSchema).max(40).default([]), + }), + transitions: z.array(EdlTransitionSchema).max(400).default([]), + targetLufs: z.number().min(-30).max(-5).optional(), + captions: EdlCaptionsSchema.optional(), + overlays: z.array(EdlOverlaySchema).max(100).default([]), + intro: MotionTemplateRefSchema.optional(), + outro: MotionTemplateRefSchema.optional(), + redactions: z + .object({ + faces: z.array(FaceRedactionSchema).max(50).default([]), + marks: z + .array(z.object({ markId: Id, treatment: MarkTreatmentSchema.exclude(["none"]) })) + .max(100) + .default([]), + regions: z.array(RegionRedactionSchema).max(100).default([]), + voice: VoiceDisguiseSchema.optional(), + }) + .default({ faces: [], marks: [], regions: [] }), + metadata: z.object({ + title: z.string().max(200).optional(), + description: z.string().max(5000).optional(), + tags: z.array(z.string().max(60)).max(50).default([]), + aiAltered: z.boolean().default(false), + aiAlterations: z.array(AiAlterationSchema).default([]), + }), +}); +export type Edl = z.infer; + +/** Machine-readable EDL validation error so an agent can correct its plan. */ +export const EdlErrorSchema = z.object({ + code: z.enum([ + "schema", + "unknown_asset", + "unknown_clip", + "unknown_artifact", + "unknown_mark", + "unknown_person", + "unknown_track", + "unknown_template", + "range_out_of_asset", + "range_empty", + "overlap", + "duration_below_min", + "duration_above_max", + "overlay_outside_safe_zone", + "paid_only_segment", + "profile_mismatch", + "music_license", + ]), + path: z.string(), + message: z.string(), +}); +export type EdlError = z.infer; + +// --------------------------------------------------------------------------- +// Render plan (compiled from an EDL; executed by MediaRenderer) +// --------------------------------------------------------------------------- + +export const RenderStepSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("ffmpeg"), + id: SafeId, + /** argv after the ffmpeg binary. Never joined into a shell string. */ + argv: z.array(z.string()), + produces: z.array(z.string()), + /** Output timeline length for progress reporting. */ + durationMs: Ms, + }), + z.object({ + kind: z.literal("motion"), + id: SafeId, + template: MotionTemplateRefSchema, + width: z.number().int(), + height: z.number().int(), + fps: z.number(), + outPath: z.string(), + }), + z.object({ + kind: z.literal("inpaint"), + id: SafeId, + markId: Id, + inputPath: z.string(), + outPath: z.string(), + }), + z.object({ + kind: z.literal("write_file"), + id: SafeId, + path: z.string(), + content: z.string(), + }), + z.object({ + kind: z.literal("synthetic_identity"), + id: SafeId, + personId: Id, + seed: z.number().int(), + inputPath: z.string(), + outPath: z.string(), + }), +]); +export type RenderStep = z.infer; + +export const RenderOutputKindSchema = z.enum(["video", "poster", "subtitles_srt", "subtitles_vtt"]); +export type RenderOutputKind = z.infer; + +export const RenderPlanSchema = z.object({ + quality: MediaRenderQualitySchema, + workDir: z.string(), + steps: z.array(RenderStepSchema), + outputs: z.array(z.object({ kind: RenderOutputKindSchema, path: z.string() })), +}); +export type RenderPlan = z.infer; + +export const FfprobeReportSchema = z.object({ + durationMs: Ms, + width: z.number().int(), + height: z.number().int(), + fps: z.number(), + videoCodec: z.string(), + audioCodec: z.string().nullable(), + sizeBytes: z.number().int(), + integratedLufs: z.number().nullable(), + truePeakDb: z.number().nullable(), + /** Container and stream tags exactly as found, for metadata-clean checks. */ + tags: z.record(z.string(), z.string()), +}); +export type FfprobeReport = z.infer; + +// --------------------------------------------------------------------------- +// Gate +// --------------------------------------------------------------------------- + +export const GateCheckSchema = z.enum([ + "technical", + "content", + "anonymization", + "consent", + "text", + "disclosure", + "brand", +]); +export type GateCheck = z.infer; + +export const GateIssueSchema = z.object({ + check: GateCheckSchema, + severity: z.enum(["block", "warn"]), + code: z.string().min(1).max(64), + message: z.string(), + atMs: Ms.optional(), + endMs: Ms.optional(), + category: ContentCategorySchema.optional(), + score: z.number().optional(), + personId: Id.optional(), + markId: Id.optional(), +}); +export type GateIssue = z.infer; + +export const GateReportSchema = z.object({ + profileId: SafeId, + passed: z.boolean(), + checkedAt: IsoDate, + sampledFrames: z.number().int().min(0), + issues: z.array(GateIssueSchema), + autoFixRound: z.number().int().min(0).max(3).default(0), +}); +export type GateReport = z.infer; + +export const MAX_AUTO_FIX_ROUNDS = 3; + +// --------------------------------------------------------------------------- +// Teaser scoring +// --------------------------------------------------------------------------- + +export const TEASER_FEATURES = [ + "motion", + "sharpness", + "faces", + "gaze", + "audioPeak", + "speechHook", + "novelty", + "beatProximity", + "aesthetics", +] as const; +export const TeaserFeatureSchema = z.enum(TEASER_FEATURES); +export type TeaserFeature = z.infer; + +export const TeaserWeightsSchema = z.record(TeaserFeatureSchema, z.number()); +export type TeaserWeights = z.infer; + +/** Minimum publications with metrics before learned weights replace defaults. */ +export const TEASER_LEARNING_MIN_SAMPLES = 20; + +// --------------------------------------------------------------------------- +// DTOs +// --------------------------------------------------------------------------- + +export const MediaSourceSchema = z.object({ + id: Id, + kind: MediaSourceKindSchema, + label: z.string(), + status: MediaSourceStatusSchema, + assetCount: z.number().int(), + lastSyncedAt: IsoDate.nullable(), + error: z.string().nullable(), + createdAt: IsoDate, +}); +export type MediaSource = z.infer; + +export const MediaAssetSchema = z.object({ + id: Id, + sourceId: Id, + kind: MediaAssetKindSchema, + storage: MediaAssetStorageSchema, + capturedAt: IsoDate.nullable(), + durationMs: Ms.nullable(), + width: z.number().int().nullable(), + height: z.number().int().nullable(), + fps: z.number().nullable(), + /** Rounded to about 1 km. */ + gpsLat: z.number().nullable(), + gpsLon: z.number().nullable(), + deviceModel: z.string().nullable(), + albumNames: z.array(z.string()), + isFavorite: z.boolean(), + ingestState: MediaIngestStateSchema, + hasProxy: z.boolean(), + hasThumbnail: z.boolean(), + qualityScore: z.number().nullable(), + createdAt: IsoDate, +}); +export type MediaAsset = z.infer; + +export const MediaSegmentSchema = z.object({ + id: Id, + assetId: Id, + kind: MediaSegmentKindSchema, + startMs: Ms, + endMs: Ms, + scores: z.partialRecord(TeaserFeatureSchema, z.number()), + teaserScore: z.number().nullable(), + transcriptExcerpt: z.string().nullable(), + labels: z.array(z.string()), + paidOnly: z.boolean(), +}); +export type MediaSegment = z.infer; + +export const MediaCollectionSchema = z.object({ + id: Id, + kind: MediaCollectionKindSchema, + title: z.string(), + labelSource: MediaLabelSourceSchema, + startAt: IsoDate.nullable(), + endAt: IsoDate.nullable(), + coverAssetId: Id.nullable(), + assetCount: z.number().int(), + videoCount: z.number().int(), + totalDurationMs: Ms, +}); +export type MediaCollection = z.infer; + +export const MediaPersonSchema = z.object({ + id: Id, + displayName: z.string(), + isSelf: z.boolean(), + confirmed: z.boolean(), + anonymizeByDefault: AnonymizeModeSchema, + consentRecordedAt: IsoDate.nullable(), + consentArtifactId: Id.nullable(), + ageVerifiedAt: IsoDate.nullable(), + ageVerificationMethod: AgeVerificationMethodSchema, + faceTrackCount: z.number().int(), + coverThumbAssetId: Id.nullable(), +}); +export type MediaPerson = z.infer; + +export const MediaFaceTrackSchema = z.object({ + id: Id, + assetId: Id, + personId: Id.nullable(), + startMs: Ms, + endMs: Ms, + confidenceMin: z.number(), +}); +export type MediaFaceTrack = z.infer; + +export const MediaMarkPromptSchema = z.object({ + tMs: Ms, + box: NormalizedBoxSchema.optional(), + point: NormalizedPointSchema.optional(), +}); +export type MediaMarkPrompt = z.infer; + +export const MediaMarkSchema = z.object({ + id: Id, + kind: MediaMarkKindSchema, + label: z.string().nullable(), + personId: Id.nullable(), + assetId: Id.nullable(), + prompt: MediaMarkPromptSchema.nullable(), + treatment: MarkTreatmentSchema, + qualityScore: z.number().nullable(), + suggestedFallback: MarkTreatmentSchema.nullable(), + createdAt: IsoDate, +}); +export type MediaMark = z.infer; + +export const BrandKitRecordSchema = z.object({ + id: Id, + kit: BrandKitSchema, + createdAt: IsoDate, + updatedAt: IsoDate, +}); +export type BrandKitRecord = z.infer; + +export const MediaProjectSchema = z.object({ + id: Id, + title: z.string(), + goal: MediaProjectGoalSchema, + status: MediaProjectStatusSchema, + brandKitId: Id.nullable(), + collectionIds: z.array(Id), + profileIds: z.array(SafeId), + createdAt: IsoDate, + updatedAt: IsoDate, +}); +export type MediaProject = z.infer; + +export const MediaEditPlanSchema = z.object({ + id: Id, + projectId: Id, + profileId: SafeId, + version: z.number().int().min(1), + edl: EdlSchema, + createdBy: z.enum(["agent", "user"]), + rationale: z.string().nullable(), + createdAt: IsoDate, +}); +export type MediaEditPlan = z.infer; + +export const MediaRenderOutputSchema = z.object({ + kind: z.enum(["video", "poster", "subtitles_srt", "subtitles_vtt", "thumbnail", "bundle"]), + /** Small outputs are artifacts; videos are streamed from the media file route. */ + artifactId: Id.nullable(), + fileName: z.string(), + sizeBytes: z.number().int(), +}); +export type MediaRenderOutput = z.infer; + +export const MediaRenderSchema = z.object({ + id: Id, + planId: Id, + projectId: Id, + profileId: SafeId, + quality: MediaRenderQualitySchema, + status: MediaRenderStatusSchema, + progress: z.number().min(0).max(1), + outputs: z.array(MediaRenderOutputSchema), + gateReport: GateReportSchema.nullable(), + ffprobe: FfprobeReportSchema.nullable(), + error: z.string().nullable(), + createdAt: IsoDate, + updatedAt: IsoDate, +}); +export type MediaRender = z.infer; + +export const UtmSchema = z.object({ + source: z.string().min(1).max(60), + medium: z.string().min(1).max(60), + campaign: z.string().min(1).max(100), + content: z.string().max(100).optional(), + term: z.string().max(100).optional(), +}); +export type Utm = z.infer; + +export const MediaPublicationSchema = z.object({ + id: Id, + renderId: Id, + profileId: SafeId, + destination: z.string(), + status: MediaPublicationStatusSchema, + scheduledAt: IsoDate.nullable(), + publishedAt: IsoDate.nullable(), + externalUrl: z.string().nullable(), + targetUrl: z.string().nullable(), + trackingUrl: z.string().nullable(), + utm: UtmSchema.nullable(), + title: z.string().nullable(), + caption: z.string().nullable(), + tags: z.array(z.string()), + thumbnailVariant: z.string().nullable(), + bundleArtifactId: Id.nullable(), + createdAt: IsoDate, +}); +export type MediaPublication = z.infer; + +export const MediaMetricSchema = z.object({ + id: Id, + publicationId: Id, + observedAt: IsoDate, + views: z.number().int().min(0), + clicks: z.number().int().min(0), + conversions: z.number().int().min(0), + revenueCents: z.number().int().min(0), + currency: z.string().length(3), + source: MediaMetricSourceSchema, +}); +export type MediaMetric = z.infer; + +export const MetricCsvMappingSchema = z.object({ + id: Id.optional(), + name: z.string().min(1).max(60), + delimiter: z.enum([",", ";", "\t"]).default(","), + columns: z.object({ + publication: z.string().min(1), + observedAt: z.string().min(1), + views: z.string().optional(), + clicks: z.string().optional(), + conversions: z.string().optional(), + revenue: z.string().optional(), + currency: z.string().optional(), + }), + /** How the publication column identifies a publication. */ + publicationKey: z.enum(["id", "tracking_url", "utm_content", "external_url"]), + revenueUnit: z.enum(["cents", "major"]).default("major"), + defaultCurrency: z.string().length(3).default("EUR"), +}); +export type MetricCsvMapping = z.infer; + +export const FunnelRowSchema = z.object({ + publicationId: Id, + projectId: Id, + profileId: SafeId, + variant: z.string().nullable(), + thumbnailVariant: z.string().nullable(), + views: z.number().int(), + clicks: z.number().int(), + conversions: z.number().int(), + revenueCents: z.number().int(), + currency: z.string(), +}); +export type FunnelRow = z.infer; + +export const MotionTemplateInfoSchema = z.object({ + id: SafeId, + kind: z.enum(["intro", "outro", "lower_third", "caption", "end_card"]), + name: z.string(), + minDurationMs: Ms, + maxDurationMs: Ms, + params: z.array( + z.object({ + name: z.string(), + type: z.enum(["string", "number", "boolean", "color"]), + required: z.boolean(), + }), + ), +}); +export type MotionTemplateInfo = z.infer; + +export const IntroVariantSchema = z.object({ + id: Id, + template: MotionTemplateRefSchema, + rationale: z.string(), + previewArtifactId: Id.nullable(), +}); +export type IntroVariant = z.infer; + +export const MarketingCopySchema = z.object({ + profileId: SafeId, + language: z.string().min(2).max(10), + title: z.string().max(200), + hook: z.string().max(200), + caption: z.string().max(5000), + hashtags: z.array(z.string().max(60)).max(50), + description: z.string().max(5000).optional(), + chapters: z.array(z.object({ atMs: Ms, title: z.string().max(100) })).optional(), + altText: z.string().max(500).optional(), + issues: z.array(GateIssueSchema), +}); +export type MarketingCopy = z.infer; + +export const MediaInferenceOperationSchema = z.enum([ + "detectScenes", + "transcribe", + "detectFaces", + "trackFaces", + "faceLandmarks", + "embedFrames", + "classifyContent", + "segmentPrompted", + "inpaintVideo", + "redact", + "synthesizeIdentity", + "audioFeatures", + "saliency", +]); +export type MediaInferenceOperation = z.infer; + +export const MediaCapabilitiesSchema = z.object({ + configured: z.boolean(), + operations: z.record( + MediaInferenceOperationSchema, + z.object({ + available: z.boolean(), + experimental: z.boolean(), + model: z.string().nullable(), + license: z.string().nullable(), + commercialUse: z.boolean().nullable(), + }), + ), + accelerator: z.enum(["cpu", "cuda", "rocm", "metal", "none"]), + hardwareEncoders: z.array(z.string()), +}); +export type MediaCapabilities = z.infer; + +/** Metadata a client-pushed source (Apple Photos, iOS library) reports per asset. */ +export const ClientMediaItemSchema = z.object({ + externalId: z.string().min(1).max(200), + kind: MediaAssetKindSchema, + capturedAt: IsoDate.nullable(), + durationMs: Ms.nullable(), + width: z.number().int().nullable(), + height: z.number().int().nullable(), + fps: z.number().nullable(), + gpsLat: z.number().min(-90).max(90).nullable(), + gpsLon: z.number().min(-180).max(180).nullable(), + deviceModel: z.string().max(100).nullable(), + albumNames: z.array(z.string().max(200)).max(50).default([]), + isFavorite: z.boolean().default(false), + fileName: z.string().max(255).nullable(), +}); +export type ClientMediaItem = z.infer; + +/** Upload chunks are sent as raw bytes to the media upload route. */ +export const MEDIA_UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024; +export const MEDIA_UPLOAD_MAX_BYTES = 20 * 1024 * 1024 * 1024; + +export const MediaUploadSchema = z.object({ + id: Id, + assetId: Id, + sizeBytes: z.number().int().positive(), + chunkBytes: z.number().int().positive(), + receivedChunks: z.array(z.number().int().min(0)), + status: z.enum(["open", "complete", "failed"]), +}); +export type MediaUpload = z.infer; + +// --------------------------------------------------------------------------- +// RPC inputs +// --------------------------------------------------------------------------- + +export const PageInput = z.object({ + cursor: z.string().max(200).nullable().default(null), + limit: z.number().int().min(1).max(500).default(100), +}); + +export const MediaAssetQuerySchema = PageInput.extend({ + text: z.string().max(200).optional(), + kinds: z.array(MediaAssetKindSchema).optional(), + from: IsoDate.optional(), + to: IsoDate.optional(), + near: z + .object({ lat: z.number(), lon: z.number(), radiusKm: z.number().positive().max(500) }) + .optional(), + personId: Id.optional(), + collectionId: Id.optional(), + sourceId: Id.optional(), + minQuality: z.number().min(0).max(1).optional(), + favoritesOnly: z.boolean().optional(), +}); +export type MediaAssetQuery = z.infer; + +export const ConsentInputSchema = z.object({ + personId: Id, + consentArtifactId: Id.nullable(), + consentRecordedAt: IsoDate, + ageVerificationMethod: AgeVerificationMethodSchema, + ageVerifiedAt: IsoDate.nullable(), +}); +export type ConsentInput = z.infer; + +export const CreateMediaProjectInput = z.object({ + title: z.string().trim().min(1).max(120), + goal: MediaProjectGoalSchema, + collectionIds: z.array(Id).max(50).default([]), + assetIds: z.array(Id).max(2000).default([]), + profileIds: z.array(SafeId).min(1).max(12), + brandKitId: Id.nullable().default(null), +}); +export type CreateMediaProjectInputType = z.infer; + +export const UpdateMediaProjectInput = z.object({ + projectId: Id, + title: z.string().trim().min(1).max(120).optional(), + goal: MediaProjectGoalSchema.optional(), + collectionIds: z.array(Id).max(50).optional(), + profileIds: z.array(SafeId).min(1).max(12).optional(), + brandKitId: Id.nullable().optional(), + status: MediaProjectStatusSchema.optional(), +}); + +export const CreateMarkInput = z.object({ + kind: MediaMarkKindSchema, + label: z.string().max(80).nullable().default(null), + personId: Id.nullable().default(null), + assetId: Id, + prompt: MediaMarkPromptSchema, + treatment: MarkTreatmentSchema, +}); + +const ok = z.object({ ok: z.literal(true) }); +const page = (item: T) => + z.object({ items: z.array(item), nextCursor: z.string().nullable() }); + +/** + * Plain-object RPC definitions. rpc.ts wraps them with `oc` so this module + * stays free of the oRPC dependency and can be imported by packages/media. + */ +export const mediaRpcShapes = { + capabilities: { input: z.object({}).default({}), output: MediaCapabilitiesSchema }, + sources: { + list: { input: z.object({}).default({}), output: z.array(MediaSourceSchema) }, + createFolder: { + input: z.object({ + label: z.string().trim().min(1).max(60), + path: z.string().min(1).max(1000), + }), + output: MediaSourceSchema, + }, + registerClient: { + input: z.object({ + kind: z.enum(CLIENT_PUSHED_MEDIA_SOURCES), + label: z.string().trim().min(1).max(60), + deviceId: z.string().min(1).max(200), + }), + output: MediaSourceSchema, + }, + pushItems: { + input: z.object({ + sourceId: Id, + items: z.array(ClientMediaItemSchema).min(1).max(500), + /** Opaque change token from the device for the next incremental push. */ + syncToken: z.string().max(2000).nullable(), + complete: z.boolean(), + }), + output: z.object({ + created: z.number().int(), + updated: z.number().int(), + unchanged: z.number().int(), + }), + }, + sync: { input: z.object({ sourceId: Id }), output: ok }, + remove: { input: z.object({ sourceId: Id }), output: ok }, + }, + assets: { + search: { input: MediaAssetQuerySchema, output: page(MediaAssetSchema) }, + get: { + input: z.object({ assetId: Id }), + output: z.object({ + asset: MediaAssetSchema, + segments: z.array(MediaSegmentSchema), + faceTracks: z.array(MediaFaceTrackSchema), + transcript: z.string().nullable(), + }), + }, + setPaidOnly: { + input: z.object({ segmentIds: z.array(Id).min(1).max(500), paidOnly: z.boolean() }), + output: ok, + }, + requestUpload: { + input: z.object({ assetIds: z.array(Id).min(1).max(2000) }), + output: z.object({ awaiting: z.array(Id) }), + }, + }, + uploads: { + begin: { + input: z.object({ + assetId: Id, + sizeBytes: z.number().int().positive().max(MEDIA_UPLOAD_MAX_BYTES), + sha256: z.string().regex(/^[0-9a-f]{64}$/), + fileName: z.string().min(1).max(255), + }), + output: MediaUploadSchema, + }, + status: { input: z.object({ uploadId: Id }), output: MediaUploadSchema }, + complete: { input: z.object({ uploadId: Id }), output: MediaUploadSchema }, + }, + collections: { + list: { + input: PageInput.extend({ kind: MediaCollectionKindSchema.optional() }), + output: page(MediaCollectionSchema), + }, + get: { + input: z.object({ collectionId: Id }), + output: z.object({ collection: MediaCollectionSchema, assets: z.array(MediaAssetSchema) }), + }, + rename: { + input: z.object({ collectionId: Id, title: z.string().trim().min(1).max(120) }), + output: MediaCollectionSchema, + }, + merge: { + input: z.object({ collectionIds: z.array(Id).min(2).max(20) }), + output: MediaCollectionSchema, + }, + split: { + input: z.object({ collectionId: Id, atAssetId: Id }), + output: z.array(MediaCollectionSchema), + }, + createManual: { + input: z.object({ + title: z.string().trim().min(1).max(120), + assetIds: z.array(Id).min(1).max(2000), + }), + output: MediaCollectionSchema, + }, + recluster: { input: z.object({}).default({}), output: ok }, + }, + people: { + list: { input: z.object({}).default({}), output: z.array(MediaPersonSchema) }, + update: { + input: z.object({ + personId: Id, + displayName: z.string().trim().min(1).max(60).optional(), + isSelf: z.boolean().optional(), + confirmed: z.boolean().optional(), + anonymizeByDefault: AnonymizeModeSchema.optional(), + }), + output: MediaPersonSchema, + }, + merge: { + input: z.object({ personIds: z.array(Id).min(2).max(20) }), + output: MediaPersonSchema, + }, + assignTrack: { + input: z.object({ trackId: Id, personId: Id.nullable() }), + output: MediaFaceTrackSchema, + }, + /** User-only. Agent tools never reach this procedure. */ + recordConsent: { input: ConsentInputSchema, output: MediaPersonSchema }, + /** Deletes the person and every biometric record tied to them. */ + remove: { input: z.object({ personId: Id }), output: ok }, + }, + marks: { + list: { + input: z.object({ personId: Id.optional(), assetId: Id.optional() }), + output: z.array(MediaMarkSchema), + }, + create: { input: CreateMarkInput, output: MediaMarkSchema }, + update: { + input: z.object({ + markId: Id, + treatment: MarkTreatmentSchema.optional(), + label: z.string().max(80).nullable().optional(), + personId: Id.nullable().optional(), + }), + output: MediaMarkSchema, + }, + remove: { input: z.object({ markId: Id }), output: ok }, + }, + brandKits: { + list: { input: z.object({}).default({}), output: z.array(BrandKitRecordSchema) }, + upsert: { + input: z.object({ id: Id.optional(), kit: BrandKitSchema }), + output: BrandKitRecordSchema, + }, + remove: { input: z.object({ id: Id }), output: ok }, + templates: { input: z.object({}).default({}), output: z.array(MotionTemplateInfoSchema) }, + introVariants: { + input: z.object({ id: Id, count: z.number().int().min(1).max(5).default(3) }), + output: z.array(IntroVariantSchema), + }, + }, + profiles: { + list: { input: z.object({}).default({}), output: z.array(PlatformProfileSummarySchema) }, + get: { input: z.object({ profileId: SafeId }), output: PlatformProfileSchema }, + }, + projects: { + list: { input: PageInput, output: page(MediaProjectSchema) }, + get: { + input: z.object({ projectId: Id }), + output: z.object({ + project: MediaProjectSchema, + plans: z.array(MediaEditPlanSchema), + renders: z.array(MediaRenderSchema), + }), + }, + create: { input: CreateMediaProjectInput, output: MediaProjectSchema }, + update: { input: UpdateMediaProjectInput, output: MediaProjectSchema }, + remove: { input: z.object({ projectId: Id }), output: ok }, + /** Deterministic first-cut plans per profile (agent refines them through media_plan). */ + autoPlan: { + input: z.object({ projectId: Id, profileIds: z.array(SafeId).max(12).optional() }), + output: z.array(MediaEditPlanSchema), + }, + }, + plans: { + submit: { + input: z.object({ + projectId: Id, + edl: z.unknown(), + rationale: z.string().max(4000).nullable().default(null), + }), + output: z.union([ + z.object({ ok: z.literal(true), plan: MediaEditPlanSchema }), + z.object({ ok: z.literal(false), errors: z.array(EdlErrorSchema) }), + ]), + }, + }, + renders: { + start: { + input: z.object({ planId: Id, quality: MediaRenderQualitySchema }), + output: MediaRenderSchema, + }, + get: { input: z.object({ renderId: Id }), output: MediaRenderSchema }, + cancel: { input: z.object({ renderId: Id }), output: MediaRenderSchema }, + thumbnails: { + input: z.object({ renderId: Id, count: z.number().int().min(1).max(6).default(3) }), + output: z.array(MediaRenderOutputSchema), + }, + }, + marketing: { + copy: { + input: z.object({ renderId: Id, language: z.string().min(2).max(10).default("en") }), + output: MarketingCopySchema, + }, + }, + publications: { + list: { + input: PageInput.extend({ projectId: Id.optional() }), + output: page(MediaPublicationSchema), + }, + create: { + input: z.object({ + renderId: Id, + destination: z.string().min(1).max(60).default("bundle"), + targetUrl: z.string().url().max(2000).nullable().default(null), + utm: UtmSchema.nullable().default(null), + title: z.string().max(200).nullable().default(null), + caption: z.string().max(5000).nullable().default(null), + tags: z.array(z.string().max(60)).max(50).default([]), + thumbnailVariant: z.string().max(40).nullable().default(null), + scheduledAt: IsoDate.nullable().default(null), + }), + output: MediaPublicationSchema, + }, + bundle: { input: z.object({ publicationId: Id }), output: MediaPublicationSchema }, + markPublished: { + input: z.object({ publicationId: Id, externalUrl: z.string().url().max(2000).nullable() }), + output: MediaPublicationSchema, + }, + cancel: { input: z.object({ publicationId: Id }), output: MediaPublicationSchema }, + }, + metrics: { + mappings: { input: z.object({}).default({}), output: z.array(MetricCsvMappingSchema) }, + saveMapping: { input: MetricCsvMappingSchema, output: MetricCsvMappingSchema }, + importCsv: { + input: z.object({ mappingId: Id, csv: z.string().min(1).max(5_000_000) }), + output: z.object({ + imported: z.number().int(), + skipped: z.array(z.object({ row: z.number().int(), reason: z.string() })), + }), + }, + record: { + input: MediaMetricSchema.omit({ id: true, source: true }), + output: MediaMetricSchema, + }, + funnel: { + input: z.object({ projectId: Id.optional() }), + output: z.array(FunnelRowSchema), + }, + }, +} as const; + +// --------------------------------------------------------------------------- +// Agent tools +// --------------------------------------------------------------------------- + +export const MEDIA_TOOL_NAMES = [ + "media_search", + "media_collections", + "media_project", + "media_plan", + "media_render", + "media_gate_report", + "media_people", + "media_marks", + "media_brand_kit", + "media_copy", + "media_thumbnails", + "media_publish", + "media_metrics", +] as const; +export type MediaToolName = (typeof MEDIA_TOOL_NAMES)[number]; + +const help = { help: z.boolean().optional() }; + +export const MediaToolInputSchemas = { + media_search: MediaAssetQuerySchema.extend({ + ...help, + themeText: z.string().max(200).optional(), + }), + media_collections: z.object({ + ...help, + action: z.enum(["list", "get", "rename", "merge", "split"]), + kind: MediaCollectionKindSchema.optional(), + collectionId: Id.optional(), + collectionIds: z.array(Id).max(20).optional(), + atAssetId: Id.optional(), + title: z.string().max(120).optional(), + }), + media_project: z.object({ + ...help, + action: z.enum(["create", "update", "get", "list", "auto_plan"]), + projectId: Id.optional(), + title: z.string().max(120).optional(), + goal: MediaProjectGoalSchema.optional(), + collectionIds: z.array(Id).max(50).optional(), + assetIds: z.array(Id).max(2000).optional(), + profileIds: z.array(SafeId).max(12).optional(), + brandKitId: Id.nullable().optional(), + }), + media_plan: z.object({ + ...help, + projectId: Id.optional(), + edl: z.unknown().optional(), + rationale: z.string().max(4000).optional(), + }), + media_render: z.object({ + ...help, + action: z.enum(["start", "status", "cancel"]), + planId: Id.optional(), + renderId: Id.optional(), + quality: MediaRenderQualitySchema.optional(), + }), + media_gate_report: z.object({ ...help, renderId: Id.optional() }), + media_people: z.object({ + ...help, + action: z.enum(["list", "update", "merge"]), + personId: Id.optional(), + personIds: z.array(Id).max(20).optional(), + displayName: z.string().max(60).optional(), + anonymizeByDefault: AnonymizeModeSchema.optional(), + }), + media_marks: z.object({ + ...help, + action: z.enum(["list", "create", "update", "remove"]), + markId: Id.optional(), + kind: MediaMarkKindSchema.optional(), + personId: Id.nullable().optional(), + assetId: Id.optional(), + prompt: MediaMarkPromptSchema.optional(), + treatment: MarkTreatmentSchema.optional(), + label: z.string().max(80).optional(), + }), + media_brand_kit: z.object({ + ...help, + action: z.enum(["list", "upsert", "intro_variants", "templates"]), + id: Id.optional(), + kit: BrandKitSchema.optional(), + count: z.number().int().min(1).max(5).optional(), + }), + media_copy: z.object({ + ...help, + renderId: Id.optional(), + language: z.string().max(10).optional(), + }), + media_thumbnails: z.object({ + ...help, + renderId: Id.optional(), + count: z.number().int().min(1).max(6).optional(), + }), + media_publish: z.object({ + ...help, + action: z.enum(["create", "bundle", "list", "cancel"]), + renderId: Id.optional(), + publicationId: Id.optional(), + projectId: Id.optional(), + targetUrl: z.string().url().max(2000).optional(), + utm: UtmSchema.optional(), + title: z.string().max(200).optional(), + caption: z.string().max(5000).optional(), + tags: z.array(z.string().max(60)).max(50).optional(), + thumbnailVariant: z.string().max(40).optional(), + scheduledAt: IsoDate.optional(), + destination: z.string().max(60).optional(), + }), + media_metrics: z.object({ + ...help, + action: z.enum(["funnel", "record"]), + projectId: Id.optional(), + metric: MediaMetricSchema.omit({ id: true, source: true }).optional(), + }), +} satisfies Record; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 6640d4c..638a943 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -82,6 +82,7 @@ import { IntegrationProviderConfigSchema, IntegrationSetupStateSchema, } from "./integration-settings.js"; +import { mediaRpcShapes } from "./media.js"; import { ModelTierMappingSchema, SetModelTierInput } from "./model-tiers.js"; import { CreateProjectGoalInput, @@ -157,6 +158,10 @@ const threadSendInput = threadTarget } }); +function rpc(shape: { input: I; output: O }) { + return oc.input(shape.input).output(shape.output); +} + export const appContract = { aiConsent: { status: oc.input(AiConsentQuerySchema).output(AiConsentStatusSchema), @@ -861,6 +866,96 @@ export const appContract = { put: oc.input(AgentSecretInputSchema).output(AgentSecretSchema), remove: oc.input(z.object({ id: Id })).output(z.object({ ok: z.literal(true) })), }, + media: { + capabilities: rpc(mediaRpcShapes.capabilities), + sources: { + list: rpc(mediaRpcShapes.sources.list), + createFolder: rpc(mediaRpcShapes.sources.createFolder), + registerClient: rpc(mediaRpcShapes.sources.registerClient), + pushItems: rpc(mediaRpcShapes.sources.pushItems), + sync: rpc(mediaRpcShapes.sources.sync), + remove: rpc(mediaRpcShapes.sources.remove), + }, + assets: { + search: rpc(mediaRpcShapes.assets.search), + get: rpc(mediaRpcShapes.assets.get), + setPaidOnly: rpc(mediaRpcShapes.assets.setPaidOnly), + requestUpload: rpc(mediaRpcShapes.assets.requestUpload), + }, + uploads: { + begin: rpc(mediaRpcShapes.uploads.begin), + status: rpc(mediaRpcShapes.uploads.status), + complete: rpc(mediaRpcShapes.uploads.complete), + }, + collections: { + list: rpc(mediaRpcShapes.collections.list), + get: rpc(mediaRpcShapes.collections.get), + rename: rpc(mediaRpcShapes.collections.rename), + merge: rpc(mediaRpcShapes.collections.merge), + split: rpc(mediaRpcShapes.collections.split), + createManual: rpc(mediaRpcShapes.collections.createManual), + recluster: rpc(mediaRpcShapes.collections.recluster), + }, + people: { + list: rpc(mediaRpcShapes.people.list), + update: rpc(mediaRpcShapes.people.update), + merge: rpc(mediaRpcShapes.people.merge), + assignTrack: rpc(mediaRpcShapes.people.assignTrack), + recordConsent: rpc(mediaRpcShapes.people.recordConsent), + remove: rpc(mediaRpcShapes.people.remove), + }, + marks: { + list: rpc(mediaRpcShapes.marks.list), + create: rpc(mediaRpcShapes.marks.create), + update: rpc(mediaRpcShapes.marks.update), + remove: rpc(mediaRpcShapes.marks.remove), + }, + brandKits: { + list: rpc(mediaRpcShapes.brandKits.list), + upsert: rpc(mediaRpcShapes.brandKits.upsert), + remove: rpc(mediaRpcShapes.brandKits.remove), + templates: rpc(mediaRpcShapes.brandKits.templates), + introVariants: rpc(mediaRpcShapes.brandKits.introVariants), + }, + profiles: { + list: rpc(mediaRpcShapes.profiles.list), + get: rpc(mediaRpcShapes.profiles.get), + }, + projects: { + list: rpc(mediaRpcShapes.projects.list), + get: rpc(mediaRpcShapes.projects.get), + create: rpc(mediaRpcShapes.projects.create), + update: rpc(mediaRpcShapes.projects.update), + remove: rpc(mediaRpcShapes.projects.remove), + autoPlan: rpc(mediaRpcShapes.projects.autoPlan), + }, + plans: { + submit: rpc(mediaRpcShapes.plans.submit), + }, + renders: { + start: rpc(mediaRpcShapes.renders.start), + get: rpc(mediaRpcShapes.renders.get), + cancel: rpc(mediaRpcShapes.renders.cancel), + thumbnails: rpc(mediaRpcShapes.renders.thumbnails), + }, + marketing: { + copy: rpc(mediaRpcShapes.marketing.copy), + }, + publications: { + list: rpc(mediaRpcShapes.publications.list), + create: rpc(mediaRpcShapes.publications.create), + bundle: rpc(mediaRpcShapes.publications.bundle), + markPublished: rpc(mediaRpcShapes.publications.markPublished), + cancel: rpc(mediaRpcShapes.publications.cancel), + }, + metrics: { + mappings: rpc(mediaRpcShapes.metrics.mappings), + saveMapping: rpc(mediaRpcShapes.metrics.saveMapping), + importCsv: rpc(mediaRpcShapes.metrics.importCsv), + record: rpc(mediaRpcShapes.metrics.record), + funnel: rpc(mediaRpcShapes.metrics.funnel), + }, + }, }; export type AppContract = typeof appContract; diff --git a/packages/db/prisma/migrations/20261003172425_media_studio/migration.sql b/packages/db/prisma/migrations/20261003172425_media_studio/migration.sql new file mode 100644 index 0000000..39e254e --- /dev/null +++ b/packages/db/prisma/migrations/20261003172425_media_studio/migration.sql @@ -0,0 +1,513 @@ +-- CreateTable +CREATE TABLE "media_sources" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "ownerUserId" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "label" TEXT NOT NULL, + "config" JSONB NOT NULL DEFAULT '{}', + "status" TEXT NOT NULL DEFAULT 'idle', + "error" TEXT, + "lastSyncCursor" TEXT, + "lastSyncedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_sources_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_assets" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "sourceId" TEXT NOT NULL, + "externalId" TEXT NOT NULL, + "revision" TEXT, + "contentHash" TEXT, + "kind" TEXT NOT NULL, + "storage" TEXT NOT NULL, + "storagePath" TEXT, + "fileName" TEXT, + "sizeBytes" BIGINT, + "capturedAt" TIMESTAMP(3), + "durationMs" INTEGER, + "width" INTEGER, + "height" INTEGER, + "fps" DOUBLE PRECISION, + "codec" TEXT, + "orientation" INTEGER, + "gpsLat" DOUBLE PRECISION, + "gpsLon" DOUBLE PRECISION, + "deviceModel" TEXT, + "albumNames" TEXT[] DEFAULT ARRAY[]::TEXT[], + "isFavorite" BOOLEAN NOT NULL DEFAULT false, + "proxyPath" TEXT, + "thumbPath" TEXT, + "spritePath" TEXT, + "qualityScore" DOUBLE PRECISION, + "analysisVersion" INTEGER NOT NULL DEFAULT 0, + "ingestState" TEXT NOT NULL DEFAULT 'indexed', + "ingestError" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_assets_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_uploads" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "assetId" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "sizeBytes" BIGINT NOT NULL, + "chunkBytes" INTEGER NOT NULL, + "sha256" TEXT NOT NULL, + "fileName" TEXT NOT NULL, + "receivedChunks" INTEGER[] DEFAULT ARRAY[]::INTEGER[], + "status" TEXT NOT NULL DEFAULT 'open', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_uploads_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_segments" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "assetId" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "startMs" INTEGER NOT NULL, + "endMs" INTEGER NOT NULL, + "scores" JSONB NOT NULL DEFAULT '{}', + "teaserScore" DOUBLE PRECISION, + "transcriptExcerpt" TEXT, + "labels" TEXT[] DEFAULT ARRAY[]::TEXT[], + "paidOnly" BOOLEAN NOT NULL DEFAULT false, + "analysisVersion" INTEGER NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "media_segments_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_analyses" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "assetId" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "modelId" TEXT NOT NULL, + "modelVersion" TEXT NOT NULL, + "data" JSONB, + "dataPath" TEXT, + "vectors" BYTEA, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "media_analyses_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_collections" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "title" TEXT NOT NULL, + "labelSource" TEXT NOT NULL DEFAULT 'rule', + "startAt" TIMESTAMP(3), + "endAt" TIMESTAMP(3), + "centroidLat" DOUBLE PRECISION, + "centroidLon" DOUBLE PRECISION, + "coverAssetId" TEXT, + "clusterKey" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_collections_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_collection_items" ( + "collectionId" TEXT NOT NULL, + "assetId" TEXT NOT NULL, + "position" INTEGER NOT NULL, + + CONSTRAINT "media_collection_items_pkey" PRIMARY KEY ("collectionId","assetId") +); + +-- CreateTable +CREATE TABLE "media_people" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "displayName" TEXT NOT NULL, + "isSelf" BOOLEAN NOT NULL DEFAULT false, + "confirmed" BOOLEAN NOT NULL DEFAULT false, + "anonymizeByDefault" TEXT NOT NULL DEFAULT 'none', + "centroid" BYTEA, + "consentArtifactId" TEXT, + "consentRecordedAt" TIMESTAMP(3), + "consentRecordedBy" TEXT, + "ageVerifiedAt" TIMESTAMP(3), + "ageVerificationMethod" TEXT NOT NULL DEFAULT 'none', + "syntheticSeed" INTEGER, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_people_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_face_tracks" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "assetId" TEXT NOT NULL, + "personId" TEXT, + "startMs" INTEGER NOT NULL, + "endMs" INTEGER NOT NULL, + "confidenceMin" DOUBLE PRECISION NOT NULL, + "keyframesPath" TEXT NOT NULL, + "embedding" BYTEA, + "modelVersion" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "media_face_tracks_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_marks" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "label" TEXT, + "personId" TEXT, + "assetId" TEXT, + "prompt" JSONB, + "treatment" TEXT NOT NULL, + "maskDir" TEXT, + "qualityScore" DOUBLE PRECISION, + "suggestedFallback" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_marks_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_brand_kits" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "kit" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_brand_kits_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_projects" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "createdBy" TEXT NOT NULL, + "title" TEXT NOT NULL, + "goal" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'draft', + "brandKitId" TEXT, + "collectionIds" TEXT[] DEFAULT ARRAY[]::TEXT[], + "assetIds" TEXT[] DEFAULT ARRAY[]::TEXT[], + "profileIds" TEXT[] DEFAULT ARRAY[]::TEXT[], + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_projects_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_edit_plans" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "profileId" TEXT NOT NULL, + "version" INTEGER NOT NULL, + "edl" JSONB NOT NULL, + "createdBy" TEXT NOT NULL, + "rationale" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "media_edit_plans_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_renders" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "planId" TEXT NOT NULL, + "profileId" TEXT NOT NULL, + "quality" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'queued', + "progress" DOUBLE PRECISION NOT NULL DEFAULT 0, + "outputs" JSONB NOT NULL DEFAULT '[]', + "gateReport" JSONB, + "ffprobe" JSONB, + "autoFixRound" INTEGER NOT NULL DEFAULT 0, + "error" TEXT, + "createdBy" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_renders_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_publications" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "renderId" TEXT NOT NULL, + "profileId" TEXT NOT NULL, + "destination" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'draft', + "scheduledAt" TIMESTAMP(3), + "publishedAt" TIMESTAMP(3), + "externalUrl" TEXT, + "targetUrl" TEXT, + "trackingUrl" TEXT, + "utm" JSONB, + "title" TEXT, + "caption" TEXT, + "tags" TEXT[] DEFAULT ARRAY[]::TEXT[], + "thumbnailVariant" TEXT, + "bundleArtifactId" TEXT, + "createdBy" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_publications_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_metrics" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "publicationId" TEXT NOT NULL, + "observedAt" TIMESTAMP(3) NOT NULL, + "views" INTEGER NOT NULL DEFAULT 0, + "clicks" INTEGER NOT NULL DEFAULT 0, + "conversions" INTEGER NOT NULL DEFAULT 0, + "revenueCents" INTEGER NOT NULL DEFAULT 0, + "currency" TEXT NOT NULL DEFAULT 'EUR', + "source" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "media_metrics_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_metric_mappings" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "mapping" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_metric_mappings_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "media_teaser_models" ( + "spaceId" TEXT NOT NULL, + "weights" JSONB NOT NULL, + "sampleCount" INTEGER NOT NULL, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_teaser_models_pkey" PRIMARY KEY ("spaceId") +); + +-- CreateIndex +CREATE INDEX "media_sources_spaceId_createdAt_idx" ON "media_sources"("spaceId", "createdAt"); + +-- CreateIndex +CREATE INDEX "media_assets_spaceId_capturedAt_idx" ON "media_assets"("spaceId", "capturedAt"); + +-- CreateIndex +CREATE INDEX "media_assets_spaceId_kind_capturedAt_idx" ON "media_assets"("spaceId", "kind", "capturedAt"); + +-- CreateIndex +CREATE INDEX "media_assets_spaceId_ingestState_idx" ON "media_assets"("spaceId", "ingestState"); + +-- CreateIndex +CREATE INDEX "media_assets_contentHash_idx" ON "media_assets"("contentHash"); + +-- CreateIndex +CREATE UNIQUE INDEX "media_assets_sourceId_externalId_key" ON "media_assets"("sourceId", "externalId"); + +-- CreateIndex +CREATE INDEX "media_uploads_assetId_idx" ON "media_uploads"("assetId"); + +-- CreateIndex +CREATE INDEX "media_uploads_spaceId_status_idx" ON "media_uploads"("spaceId", "status"); + +-- CreateIndex +CREATE INDEX "media_segments_assetId_startMs_idx" ON "media_segments"("assetId", "startMs"); + +-- CreateIndex +CREATE INDEX "media_segments_spaceId_teaserScore_idx" ON "media_segments"("spaceId", "teaserScore"); + +-- CreateIndex +CREATE INDEX "media_analyses_spaceId_kind_idx" ON "media_analyses"("spaceId", "kind"); + +-- CreateIndex +CREATE UNIQUE INDEX "media_analyses_assetId_kind_modelVersion_key" ON "media_analyses"("assetId", "kind", "modelVersion"); + +-- CreateIndex +CREATE INDEX "media_collections_spaceId_kind_startAt_idx" ON "media_collections"("spaceId", "kind", "startAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "media_collections_spaceId_kind_clusterKey_key" ON "media_collections"("spaceId", "kind", "clusterKey"); + +-- CreateIndex +CREATE INDEX "media_collection_items_assetId_idx" ON "media_collection_items"("assetId"); + +-- CreateIndex +CREATE INDEX "media_people_spaceId_idx" ON "media_people"("spaceId"); + +-- CreateIndex +CREATE INDEX "media_face_tracks_assetId_idx" ON "media_face_tracks"("assetId"); + +-- CreateIndex +CREATE INDEX "media_face_tracks_personId_idx" ON "media_face_tracks"("personId"); + +-- CreateIndex +CREATE INDEX "media_marks_spaceId_personId_idx" ON "media_marks"("spaceId", "personId"); + +-- CreateIndex +CREATE INDEX "media_marks_assetId_idx" ON "media_marks"("assetId"); + +-- CreateIndex +CREATE INDEX "media_brand_kits_spaceId_idx" ON "media_brand_kits"("spaceId"); + +-- CreateIndex +CREATE INDEX "media_projects_spaceId_updatedAt_idx" ON "media_projects"("spaceId", "updatedAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "media_edit_plans_projectId_profileId_version_key" ON "media_edit_plans"("projectId", "profileId", "version"); + +-- CreateIndex +CREATE INDEX "media_renders_projectId_createdAt_idx" ON "media_renders"("projectId", "createdAt"); + +-- CreateIndex +CREATE INDEX "media_renders_spaceId_status_idx" ON "media_renders"("spaceId", "status"); + +-- CreateIndex +CREATE INDEX "media_publications_spaceId_createdAt_idx" ON "media_publications"("spaceId", "createdAt"); + +-- CreateIndex +CREATE INDEX "media_publications_renderId_idx" ON "media_publications"("renderId"); + +-- CreateIndex +CREATE INDEX "media_metrics_spaceId_observedAt_idx" ON "media_metrics"("spaceId", "observedAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "media_metrics_publicationId_observedAt_source_key" ON "media_metrics"("publicationId", "observedAt", "source"); + +-- CreateIndex +CREATE UNIQUE INDEX "media_metric_mappings_spaceId_name_key" ON "media_metric_mappings"("spaceId", "name"); + +-- AddForeignKey +ALTER TABLE "media_sources" ADD CONSTRAINT "media_sources_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_assets" ADD CONSTRAINT "media_assets_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_assets" ADD CONSTRAINT "media_assets_sourceId_fkey" FOREIGN KEY ("sourceId") REFERENCES "media_sources"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_uploads" ADD CONSTRAINT "media_uploads_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_uploads" ADD CONSTRAINT "media_uploads_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "media_assets"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_segments" ADD CONSTRAINT "media_segments_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_segments" ADD CONSTRAINT "media_segments_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "media_assets"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_analyses" ADD CONSTRAINT "media_analyses_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_analyses" ADD CONSTRAINT "media_analyses_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "media_assets"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_collections" ADD CONSTRAINT "media_collections_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_collection_items" ADD CONSTRAINT "media_collection_items_collectionId_fkey" FOREIGN KEY ("collectionId") REFERENCES "media_collections"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_collection_items" ADD CONSTRAINT "media_collection_items_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "media_assets"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_people" ADD CONSTRAINT "media_people_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_face_tracks" ADD CONSTRAINT "media_face_tracks_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_face_tracks" ADD CONSTRAINT "media_face_tracks_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "media_assets"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_face_tracks" ADD CONSTRAINT "media_face_tracks_personId_fkey" FOREIGN KEY ("personId") REFERENCES "media_people"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_marks" ADD CONSTRAINT "media_marks_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_marks" ADD CONSTRAINT "media_marks_personId_fkey" FOREIGN KEY ("personId") REFERENCES "media_people"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_marks" ADD CONSTRAINT "media_marks_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "media_assets"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_brand_kits" ADD CONSTRAINT "media_brand_kits_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_projects" ADD CONSTRAINT "media_projects_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_projects" ADD CONSTRAINT "media_projects_brandKitId_fkey" FOREIGN KEY ("brandKitId") REFERENCES "media_brand_kits"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_edit_plans" ADD CONSTRAINT "media_edit_plans_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_edit_plans" ADD CONSTRAINT "media_edit_plans_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "media_projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_renders" ADD CONSTRAINT "media_renders_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_renders" ADD CONSTRAINT "media_renders_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "media_projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_renders" ADD CONSTRAINT "media_renders_planId_fkey" FOREIGN KEY ("planId") REFERENCES "media_edit_plans"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_publications" ADD CONSTRAINT "media_publications_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_publications" ADD CONSTRAINT "media_publications_renderId_fkey" FOREIGN KEY ("renderId") REFERENCES "media_renders"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_metrics" ADD CONSTRAINT "media_metrics_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_metrics" ADD CONSTRAINT "media_metrics_publicationId_fkey" FOREIGN KEY ("publicationId") REFERENCES "media_publications"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_metric_mappings" ADD CONSTRAINT "media_metric_mappings_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "media_teaser_models" ADD CONSTRAINT "media_teaser_models_spaceId_fkey" FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index a6dc085..ebdd7b9 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -216,6 +216,23 @@ model Space { userFactVisibility UserFactSpace[] delegatedTasks DelegatedTask[] semanticMemoryItems SemanticMemoryItem[] + mediaSources MediaSource[] + mediaAssets MediaAsset[] + mediaUploads MediaUpload[] + mediaSegments MediaSegment[] + mediaAnalyses MediaAnalysis[] + mediaCollections MediaCollection[] + mediaPeople MediaPerson[] + mediaFaceTracks MediaFaceTrack[] + mediaMarks MediaMark[] + mediaBrandKits MediaBrandKit[] + mediaProjects MediaProject[] + mediaEditPlans MediaEditPlan[] + mediaRenders MediaRender[] + mediaPublications MediaPublication[] + mediaMetrics MediaMetric[] + mediaMetricMappings MediaMetricMapping[] + mediaTeaserModel MediaTeaserModel? @@unique([id, organizationId]) @@index([organizationId, createdAt]) @@ -1497,3 +1514,411 @@ model DelegatedTask { // CHECK constraint delegated_tasks_status_chk is created in SQL. @@map("delegated_tasks") } + +// --------------------------------------------------------------------------- +// Media Studio +// --------------------------------------------------------------------------- + +model MediaSource { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + ownerUserId String + kind String + label String + /// Folder: { path }. Client-pushed: { deviceId }. Never secrets. + config Json @default("{}") + status String @default("idle") + error String? + lastSyncCursor String? + lastSyncedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + assets MediaAsset[] + + @@index([spaceId, createdAt]) + @@map("media_sources") +} + +model MediaAsset { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + sourceId String + source MediaSource @relation(fields: [sourceId], references: [id], onDelete: Cascade) + externalId String + revision String? + contentHash String? + kind String + storage String + /// Uploaded originals: path relative to the media root. Referenced originals resolve through the source. + storagePath String? + fileName String? + sizeBytes BigInt? + capturedAt DateTime? + durationMs Int? + width Int? + height Int? + fps Float? + codec String? + orientation Int? + /// Rounded to two decimals (about 1 km). + gpsLat Float? + gpsLon Float? + deviceModel String? + albumNames String[] @default([]) + isFavorite Boolean @default(false) + proxyPath String? + thumbPath String? + spritePath String? + qualityScore Float? + analysisVersion Int @default(0) + ingestState String @default("indexed") + ingestError String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + segments MediaSegment[] + analyses MediaAnalysis[] + collectionItems MediaCollectionItem[] + faceTracks MediaFaceTrack[] + marks MediaMark[] + uploads MediaUpload[] + + @@unique([sourceId, externalId]) + @@index([spaceId, capturedAt]) + @@index([spaceId, kind, capturedAt]) + @@index([spaceId, ingestState]) + @@index([contentHash]) + @@map("media_assets") +} + +model MediaUpload { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + assetId String + asset MediaAsset @relation(fields: [assetId], references: [id], onDelete: Cascade) + userId String + sizeBytes BigInt + chunkBytes Int + sha256 String + fileName String + receivedChunks Int[] @default([]) + status String @default("open") + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([assetId]) + @@index([spaceId, status]) + @@map("media_uploads") +} + +model MediaSegment { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + assetId String + asset MediaAsset @relation(fields: [assetId], references: [id], onDelete: Cascade) + kind String + startMs Int + endMs Int + /// Partial> + scores Json @default("{}") + teaserScore Float? + transcriptExcerpt String? + labels String[] @default([]) + paidOnly Boolean @default(false) + analysisVersion Int + createdAt DateTime @default(now()) + + @@index([assetId, startMs]) + @@index([spaceId, teaserScore]) + @@map("media_segments") +} + +model MediaAnalysis { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + assetId String + asset MediaAsset @relation(fields: [assetId], references: [id], onDelete: Cascade) + kind String + modelId String + modelVersion String + /// Small results inline; large results as a path relative to the media root. + data Json? + dataPath String? + /// Float32 little-endian vectors for embeddings. + vectors Bytes? + createdAt DateTime @default(now()) + + @@unique([assetId, kind, modelVersion]) + @@index([spaceId, kind]) + @@map("media_analyses") +} + +model MediaCollection { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + kind String + title String + labelSource String @default("rule") + startAt DateTime? + endAt DateTime? + centroidLat Float? + centroidLon Float? + coverAssetId String? + /// Stable key so re-clustering updates instead of recreating user-renamed collections. + clusterKey String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + items MediaCollectionItem[] + + @@unique([spaceId, kind, clusterKey]) + @@index([spaceId, kind, startAt]) + @@map("media_collections") +} + +model MediaCollectionItem { + collectionId String + collection MediaCollection @relation(fields: [collectionId], references: [id], onDelete: Cascade) + assetId String + asset MediaAsset @relation(fields: [assetId], references: [id], onDelete: Cascade) + position Int + + @@id([collectionId, assetId]) + @@index([assetId]) + @@map("media_collection_items") +} + +model MediaPerson { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + displayName String + isSelf Boolean @default(false) + confirmed Boolean @default(false) + anonymizeByDefault String @default("none") + /// Mean identity embedding (Float32). Biometric: local only, deleted with the person. + centroid Bytes? + consentArtifactId String? + consentRecordedAt DateTime? + consentRecordedBy String? + ageVerifiedAt DateTime? + ageVerificationMethod String @default("none") + /// Seed for a stable synthetic identity. + syntheticSeed Int? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + faceTracks MediaFaceTrack[] + marks MediaMark[] + + @@index([spaceId]) + @@map("media_people") +} + +model MediaFaceTrack { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + assetId String + asset MediaAsset @relation(fields: [assetId], references: [id], onDelete: Cascade) + personId String? + person MediaPerson? @relation(fields: [personId], references: [id], onDelete: SetNull) + startMs Int + endMs Int + confidenceMin Float + /// Box keyframes JSON relative to the media root. + keyframesPath String + /// Biometric (Float32). Local only. + embedding Bytes? + modelVersion String + createdAt DateTime @default(now()) + + @@index([assetId]) + @@index([personId]) + @@map("media_face_tracks") +} + +model MediaMark { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + kind String + label String? + personId String? + person MediaPerson? @relation(fields: [personId], references: [id], onDelete: Cascade) + assetId String? + asset MediaAsset? @relation(fields: [assetId], references: [id], onDelete: Cascade) + /// MediaMarkPrompt + prompt Json? + treatment String + maskDir String? + qualityScore Float? + suggestedFallback String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([spaceId, personId]) + @@index([assetId]) + @@map("media_marks") +} + +model MediaBrandKit { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + /// BrandKit + kit Json + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + projects MediaProject[] + + @@index([spaceId]) + @@map("media_brand_kits") +} + +model MediaProject { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + createdBy String + title String + goal String + status String @default("draft") + brandKitId String? + brandKit MediaBrandKit? @relation(fields: [brandKitId], references: [id], onDelete: SetNull) + collectionIds String[] @default([]) + assetIds String[] @default([]) + profileIds String[] @default([]) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + plans MediaEditPlan[] + renders MediaRender[] + + @@index([spaceId, updatedAt]) + @@map("media_projects") +} + +model MediaEditPlan { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + projectId String + project MediaProject @relation(fields: [projectId], references: [id], onDelete: Cascade) + profileId String + version Int + /// Edl, validated before insert. + edl Json + createdBy String + rationale String? + createdAt DateTime @default(now()) + renders MediaRender[] + + @@unique([projectId, profileId, version]) + @@map("media_edit_plans") +} + +model MediaRender { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + projectId String + project MediaProject @relation(fields: [projectId], references: [id], onDelete: Cascade) + planId String + plan MediaEditPlan @relation(fields: [planId], references: [id], onDelete: Cascade) + profileId String + quality String + status String @default("queued") + progress Float @default(0) + /// MediaRenderOutput[] plus internal relative paths. + outputs Json @default("[]") + gateReport Json? + ffprobe Json? + autoFixRound Int @default(0) + error String? + createdBy String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + publications MediaPublication[] + + @@index([projectId, createdAt]) + @@index([spaceId, status]) + @@map("media_renders") +} + +model MediaPublication { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + renderId String + render MediaRender @relation(fields: [renderId], references: [id], onDelete: Cascade) + profileId String + destination String + status String @default("draft") + scheduledAt DateTime? + publishedAt DateTime? + externalUrl String? + targetUrl String? + trackingUrl String? + utm Json? + title String? + caption String? + tags String[] @default([]) + thumbnailVariant String? + bundleArtifactId String? + createdBy String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + metrics MediaMetric[] + + @@index([spaceId, createdAt]) + @@index([renderId]) + @@map("media_publications") +} + +model MediaMetric { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + publicationId String + publication MediaPublication @relation(fields: [publicationId], references: [id], onDelete: Cascade) + observedAt DateTime + views Int @default(0) + clicks Int @default(0) + conversions Int @default(0) + revenueCents Int @default(0) + currency String @default("EUR") + source String + createdAt DateTime @default(now()) + + @@unique([publicationId, observedAt, source]) + @@index([spaceId, observedAt]) + @@map("media_metrics") +} + +model MediaMetricMapping { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + name String + /// MetricCsvMapping without id + mapping Json + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([spaceId, name]) + @@map("media_metric_mappings") +} + +model MediaTeaserModel { + spaceId String @id + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + /// TeaserWeights + weights Json + sampleCount Int + updatedAt DateTime @updatedAt + + @@map("media_teaser_models") +} diff --git a/packages/media/package.json b/packages/media/package.json new file mode 100644 index 0000000..ab42765 --- /dev/null +++ b/packages/media/package.json @@ -0,0 +1,30 @@ +{ + "name": "@rakazo/media", + "version": "0.1.0", + "license": "Apache-2.0", + "type": "module", + "exports": { + ".": { + "types": "./src/index.ts", + "development": "./src/index.ts", + "default": "./src/index.ts" + }, + "./node": { + "types": "./src/node.ts", + "development": "./src/node.ts", + "default": "./src/node.ts" + } + }, + "scripts": { + "check": "tsc --noEmit -p tsconfig.json", + "test": "vitest run --root ../.. packages/media/src" + }, + "dependencies": { + "@rakazo/contracts": "workspace:*", + "zod": "4.6.2" + }, + "devDependencies": { + "typescript": "7.0.2", + "vitest": "4.1.11" + } +} diff --git a/packages/media/src/clustering.ts b/packages/media/src/clustering.ts new file mode 100644 index 0000000..59b696e --- /dev/null +++ b/packages/media/src/clustering.ts @@ -0,0 +1,87 @@ +import type { MediaAssetKind } from "@rakazo/contracts"; +import { notImplemented } from "./not-implemented.js"; + +export interface ClusterAsset { + id: string; + kind: MediaAssetKind; + capturedAt: Date | null; + gpsLat: number | null; + gpsLon: number | null; + deviceModel: string | null; + albumNames: string[]; + /** Mean frame embedding; null when not analyzed yet. */ + embedding: Float32Array | null; +} + +export interface SessionOptions { + /** New session after a gap longer than this. Default 45. */ + gapMinutes: number; + /** New session after a location jump larger than this. Default 2. */ + maxDistanceKm: number; + /** New session on calendar-day change when the gap exceeds this. Default 3. */ + dayBreakHours: number; + /** Merge adjacent sessions whose mean embeddings reach this cosine. Default 0.85. */ + mergeSimilarity: number; + /** ...and whose gap is below this. Default 2. */ + mergeMaxGapHours: number; + /** IANA zone for calendar-day boundaries. Default "UTC". */ + timeZone: string; +} + +export const DEFAULT_SESSION_OPTIONS: SessionOptions = { + gapMinutes: 45, + maxDistanceKm: 2, + dayBreakHours: 3, + mergeSimilarity: 0.85, + mergeMaxGapHours: 2, + timeZone: "UTC", +}; + +export interface SessionCluster { + /** Stable across re-clustering: derived from the earliest asset id of the session. */ + key: string; + assetIds: string[]; + startAt: Date | null; + endAt: Date | null; + centroid: { lat: number; lon: number } | null; +} + +/** + * Deterministic session grouping (spec 4.4). Assets without capturedAt form + * one trailing "undated" session; exact duplicates (same id) are ignored. + */ +export function buildSessions( + _assets: ClusterAsset[], + _options?: Partial, +): SessionCluster[] { + return notImplemented("buildSessions"); +} + +export interface VectorItem { + id: string; + vector: Float32Array; +} + +/** Agglomerative average-linkage clustering; returns groups with stable keys. */ +export function clusterThemes( + _items: VectorItem[], + _options?: { similarity?: number; minSize?: number }, +): Array<{ key: string; ids: string[] }> { + return notImplemented("clusterThemes"); +} + +/** Group face tracks into person suggestions by identity embedding. Local only. */ +export function clusterFaces( + _tracks: VectorItem[], + _options?: { similarity?: number }, +): Array<{ key: string; ids: string[]; centroid: Float32Array }> { + return notImplemented("clusterFaces"); +} + +/** Rule-based title from date range, place centroid, and dominant album. No model involved. */ +export function ruleLabel( + _cluster: { startAt: Date | null; endAt: Date | null; albumNames: string[] }, + _locale?: string, +): string { + return notImplemented("ruleLabel"); +} diff --git a/packages/media/src/compile.ts b/packages/media/src/compile.ts new file mode 100644 index 0000000..913e831 --- /dev/null +++ b/packages/media/src/compile.ts @@ -0,0 +1,117 @@ +import type { + BrandKit, + Edl, + MediaRenderQuality, + NormalizedBox, + PlatformProfile, + ReframeKeyframe, + RenderPlan, +} from "@rakazo/contracts"; +import { notImplemented } from "./not-implemented.js"; + +export interface CompileSource { + /** Redacted intermediate when redactions apply, otherwise original or proxy. */ + path: string; + width: number; + height: number; + durationMs: number; + fps: number; + hasAudio: boolean; + rotation: 0 | 90 | 180 | 270; +} + +export interface CaptionWord { + startMs: number; + endMs: number; + text: string; +} + +export interface CompileInput { + edl: Edl; + profile: PlatformProfile; + brandKit: BrandKit | null; + quality: MediaRenderQuality; + /** Absolute work directory for intermediates and outputs. */ + workDir: string; + sources: Map; + /** Source-time words per asset for captions. */ + transcripts: Map; + /** Saliency or subject points per asset for "auto" reframe. */ + subjects: Map>; + /** Pre-rendered motion clips keyed by overlay id, "intro", "outro". */ + motionClips: Map; + files: { + lutPath: string | null; + watermarkPath: string | null; + fonts: Map; + /** artifactId → absolute path for audio clips. */ + audio: Map; + }; + /** ffmpeg video encoder name, e.g. libx264, h264_vaapi, h264_videotoolbox. */ + videoEncoder: string; +} + +/** + * Pure compiler from a validated EDL to ffmpeg argv steps plus caption files. + * No string ever reaches a shell; text is escaped for filtergraph and ASS, + * every label comes from SafeId values. + */ +export function compileRenderPlan(_input: CompileInput): RenderPlan { + return notImplemented("compileRenderPlan"); +} + +export interface CaptionCue { + startMs: number; + endMs: number; + text: string; + /** Word timings inside the cue for karaoke highlight. */ + words: CaptionWord[]; +} + +export function buildCaptionCues( + _words: CaptionWord[], + _options: { maxCharsPerLine: number; maxLines: number }, +): CaptionCue[] { + return notImplemented("buildCaptionCues"); +} + +export function toSrt(_cues: CaptionCue[]): string { + return notImplemented("toSrt"); +} + +export function toVtt(_cues: CaptionCue[]): string { + return notImplemented("toVtt"); +} + +/** Smoothed crop path that keeps the subject in frame for the output aspect. */ +export function autoReframe( + _subjects: Array<{ tMs: number; cx: number; cy: number; strength: number }>, + _input: { + sourceWidth: number; + sourceHeight: number; + outWidth: number; + outHeight: number; + durationMs: number; + }, +): ReframeKeyframe[] { + return notImplemented("autoReframe"); +} + +/** Crop path that keeps every given face box outside the output frame, or null if impossible. */ +export function reframeExcluding( + _faces: Array<{ tMs: number; box: NormalizedBox }>, + _input: { + sourceWidth: number; + sourceHeight: number; + outWidth: number; + outHeight: number; + durationMs: number; + }, +): ReframeKeyframe[] | null { + return notImplemented("reframeExcluding"); +} + +/** Escape text for ffmpeg drawtext/filtergraph values. */ +export function escapeFilterText(_text: string): string { + return notImplemented("escapeFilterText"); +} diff --git a/packages/media/src/edl.ts b/packages/media/src/edl.ts new file mode 100644 index 0000000..4c3a06e --- /dev/null +++ b/packages/media/src/edl.ts @@ -0,0 +1,45 @@ +import type { BrandKit, Edl, EdlError, MediaAssetKind, PlatformProfile } from "@rakazo/contracts"; +import { notImplemented } from "./not-implemented.js"; + +export interface EdlContext { + profile: PlatformProfile; + brandKit: BrandKit | null; + assets: Map< + string, + { kind: MediaAssetKind; durationMs: number | null; width: number | null; height: number | null } + >; + /** Artifact ids the plan may reference (music, voice-over, fonts, logos). */ + artifactIds: Set; + /** Music artifacts and whether their license allows commercial use. */ + music: Map; + markIds: Set; + personIds: Set; + faceTrackIds: Set; + templateIds: Set; + /** Segments the user reserved for the paid release. */ + paidOnly: Array<{ assetId: string; startMs: number; endMs: number }>; +} + +export type EdlValidation = { ok: true; edl: Edl } | { ok: false; errors: EdlError[] }; + +/** + * Parse with EdlSchema, then check references, ranges, overlaps, durations, + * safe zones, paid-only use (teaser-style profiles), and music licensing. + * Every failure is reported; nothing is silently corrected. + */ +export function validateEdl(_input: unknown, _context: EdlContext): EdlValidation { + return notImplemented("validateEdl"); +} + +/** Output length in ms including intro, outro, speed, and transition overlap. */ +export function edlDurationMs(_edl: Edl): number { + return notImplemented("edlDurationMs"); +} + +/** Map an output timestamp back to the clip and source time it shows. */ +export function locateOutputTime( + _edl: Edl, + _outputMs: number, +): { clipId: string; assetId: string; sourceMs: number } | null { + return notImplemented("locateOutputTime"); +} diff --git a/packages/media/src/gate.ts b/packages/media/src/gate.ts new file mode 100644 index 0000000..f3e4610 --- /dev/null +++ b/packages/media/src/gate.ts @@ -0,0 +1,87 @@ +import type { + AiAlteration, + ContentCategory, + Edl, + FfprobeReport, + GateIssue, + GateReport, + NormalizedBox, + PlatformProfile, +} from "@rakazo/contracts"; +import { notImplemented } from "./not-implemented.js"; + +/** Tags a clean export may carry. Everything else is a metadata leak. */ +export const METADATA_ALLOWED_TAGS: readonly string[] = []; + +export interface GateInput { + profile: PlatformProfile; + edl: Edl; + ffprobe: FfprobeReport; + /** Classifier scores on the rendered output. */ + content: Array<{ tMs: number; scores: Partial> }>; + /** + * Faces re-detected on the rendered output. personId null means the face + * could not be matched to a known person. + */ + faces: Array<{ tMs: number; box: NormalizedBox; personId: string | null; coverage: number }>; + /** Frames where tracking lost a face that must stay anonymized for longer than allowed. */ + trackingGaps: Array<{ startMs: number; endMs: number; personId: string | null }>; + /** Residual visibility 0..1 of treated marks in the output. */ + marks: Array<{ markId: string; tMs: number; residualVisibility: number }>; + /** People visible in the output with their protection and consent state. */ + people: Array<{ + id: string; + anonymizeByDefault: string; + consentRecordedAt: string | null; + ageVerifiedAt: string | null; + }>; + copy: { title: string | null; caption: string | null; tags: string[] }; + aiAlterations: AiAlteration[]; + bundleHasDisclosure: boolean; + watermarkBox: NormalizedBox | null; + sampledFrames: number; + autoFixRound: number; + now: Date; +} + +/** Deterministic gate (spec 6). Any blocking issue fails the report. */ +export function evaluateGate(_input: GateInput): GateReport { + return notImplemented("evaluateGate"); +} + +export function checkTechnical( + _profile: PlatformProfile, + _edl: Edl, + _probe: FfprobeReport, +): GateIssue[] { + return notImplemented("checkTechnical"); +} + +export function findMetadataLeaks(_tags: Record): string[] { + return notImplemented("findMetadataLeaks"); +} + +export function checkCopy( + _profile: PlatformProfile, + _copy: { title: string | null; caption: string | null; tags: string[] }, +): GateIssue[] { + return notImplemented("checkCopy"); +} + +export interface AutoFixResult { + edl: Edl; + changes: string[]; +} + +/** + * Compliance-only repairs: trim or drop offending clips, change crop, widen + * redaction padding, swap to another segment. Never adds obfuscation aimed at + * classifiers. Returns null when the round limit is reached or nothing safe applies. + */ +export function planAutoFix( + _edl: Edl, + _report: GateReport, + _alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>, +): AutoFixResult | null { + return notImplemented("planAutoFix"); +} diff --git a/packages/media/src/geo.ts b/packages/media/src/geo.ts new file mode 100644 index 0000000..2efc882 --- /dev/null +++ b/packages/media/src/geo.ts @@ -0,0 +1,28 @@ +import { notImplemented } from "./not-implemented.js"; + +/** Great-circle distance in kilometres. */ +export function haversineKm( + _a: { lat: number; lon: number }, + _b: { lat: number; lon: number }, +): number { + return notImplemented("haversineKm"); +} + +/** Round a coordinate to two decimals (about 1 km) before storage or display. */ +export function roundCoordinate(_value: number): number { + return notImplemented("roundCoordinate"); +} + +/** Cosine similarity of two equal-length vectors. */ +export function cosineSimilarity(_a: ArrayLike, _b: ArrayLike): number { + return notImplemented("cosineSimilarity"); +} + +/** Encode Float32 vectors as little-endian bytes for MediaAnalysis.vectors and centroids. */ +export function encodeVectors(_vectors: number[][]): Uint8Array { + return notImplemented("encodeVectors"); +} + +export function decodeVectors(_bytes: Uint8Array, _dim: number): Float32Array[] { + return notImplemented("decodeVectors"); +} diff --git a/packages/media/src/index.ts b/packages/media/src/index.ts new file mode 100644 index 0000000..c1111ff --- /dev/null +++ b/packages/media/src/index.ts @@ -0,0 +1,8 @@ +export * from "./clustering.js"; +export * from "./compile.js"; +export * from "./edl.js"; +export * from "./gate.js"; +export * from "./geo.js"; +export * from "./marketing.js"; +export * from "./profiles.js"; +export * from "./teaser.js"; diff --git a/packages/media/src/marketing.ts b/packages/media/src/marketing.ts new file mode 100644 index 0000000..e63db11 --- /dev/null +++ b/packages/media/src/marketing.ts @@ -0,0 +1,66 @@ +import type { + BrandKit, + MetricCsvMapping, + MotionTemplateInfo, + MotionTemplateRef, + PlatformProfile, + Utm, +} from "@rakazo/contracts"; +import { notImplemented } from "./not-implemented.js"; + +/** Append UTM parameters to an http(s) target. Rejects other schemes. */ +export function buildTrackingUrl(_targetUrl: string, _utm: Utm): string { + return notImplemented("buildTrackingUrl"); +} + +export function normalizeHashtags(_tags: string[], _limit: number): string[] { + return notImplemented("normalizeHashtags"); +} + +export interface ParsedMetricRow { + row: number; + publicationKey: string; + observedAt: Date; + views: number; + clicks: number; + conversions: number; + revenueCents: number; + currency: string; +} + +/** RFC 4180 CSV parsing with the saved column mapping. */ +export function parseMetricsCsv( + _csv: string, + _mapping: MetricCsvMapping, +): { rows: ParsedMetricRow[]; skipped: Array<{ row: number; reason: string }> } { + return notImplemented("parseMetricsCsv"); +} + +/** Whether the brand intro fits before the hook window; shorts put the hook first. */ +export function introFits(_profile: PlatformProfile, _introMs: number): boolean { + return notImplemented("introFits"); +} + +/** Deterministic intro proposals from the brand kit, tone, and available templates. */ +export function proposeIntroVariants( + _kit: BrandKit, + _templates: MotionTemplateInfo[], + _count: number, +): Array<{ template: MotionTemplateRef; rationale: string }> { + return notImplemented("proposeIntroVariants"); +} + +/** Structural brand-kit checks beyond the schema (contrast, duplicate presets). */ +export function validateBrandKit(_kit: BrandKit): string[] { + return notImplemented("validateBrandKit"); +} + +/** Upload checklist markdown for an export bundle. */ +export function uploadChecklist(_input: { + profile: PlatformProfile; + aiAltered: boolean; + hasCaptions: boolean; + trackingUrl: string | null; +}): string { + return notImplemented("uploadChecklist"); +} diff --git a/packages/media/src/node.ts b/packages/media/src/node.ts new file mode 100644 index 0000000..5d6911a --- /dev/null +++ b/packages/media/src/node.ts @@ -0,0 +1,18 @@ +import type { PlatformProfile } from "@rakazo/contracts"; +import { notImplemented } from "./not-implemented.js"; +import type { ProfileLoadError } from "./profiles.js"; + +/** Directory of the profile JSON files shipped with this package. */ +export const BUILTIN_PROFILES_DIR = new URL("../profiles/", import.meta.url); + +/** Read every *.json in a directory; missing directory yields no profiles. */ +export async function loadProfilesFromDir( + _dir: string | URL, +): Promise<{ profiles: PlatformProfile[]; errors: ProfileLoadError[] }> { + return notImplemented("loadProfilesFromDir"); +} + +/** Resolve a path inside root; rejects traversal and symlinks that escape it. */ +export async function resolveInside(_root: string, _relative: string): Promise { + return notImplemented("resolveInside"); +} diff --git a/packages/media/src/not-implemented.ts b/packages/media/src/not-implemented.ts new file mode 100644 index 0000000..0c4ab30 --- /dev/null +++ b/packages/media/src/not-implemented.ts @@ -0,0 +1,4 @@ +/** Phase A scaffold marker. No call site may remain once Media Studio ships. */ +export function notImplemented(name: string): never { + throw new Error(`not implemented: ${name}`); +} diff --git a/packages/media/src/profiles.ts b/packages/media/src/profiles.ts new file mode 100644 index 0000000..c78fe82 --- /dev/null +++ b/packages/media/src/profiles.ts @@ -0,0 +1,38 @@ +import type { PlatformProfile, PlatformProfileSummary } from "@rakazo/contracts"; +import { notImplemented } from "./not-implemented.js"; + +export const PROFILE_STALE_DAYS = 90; + +export interface ProfileLoadError { + file: string; + message: string; +} + +export interface ProfileSet { + profiles: Map; + errors: ProfileLoadError[]; +} + +/** Parse raw JSON documents; invalid ones become errors and are skipped whole. */ +export function parseProfiles(_documents: Array<{ file: string; json: unknown }>): { + profiles: PlatformProfile[]; + errors: ProfileLoadError[]; +} { + return notImplemented("parseProfiles"); +} + +/** Private profiles override builtin ones with the same id. */ +export function mergeProfiles( + _builtin: PlatformProfile[], + _private: PlatformProfile[], +): ProfileSet { + return notImplemented("mergeProfiles"); +} + +export function summarizeProfile( + _profile: PlatformProfile, + _origin: "builtin" | "private", + _now: Date, +): PlatformProfileSummary { + return notImplemented("summarizeProfile"); +} diff --git a/packages/media/src/teaser.ts b/packages/media/src/teaser.ts new file mode 100644 index 0000000..d52418f --- /dev/null +++ b/packages/media/src/teaser.ts @@ -0,0 +1,98 @@ +import type { + BrandKit, + Edl, + MediaProjectGoal, + PlatformProfile, + TeaserFeature, + TeaserWeights, +} from "@rakazo/contracts"; +import { notImplemented } from "./not-implemented.js"; + +export type SegmentFeatures = Partial>; + +export function defaultTeaserWeights(_goal: MediaProjectGoal): TeaserWeights { + return notImplemented("defaultTeaserWeights"); +} + +export interface ScoredSegmentInput { + id: string; + assetId: string; + startMs: number; + endMs: number; + capturedAt: Date | null; + features: SegmentFeatures; + paidOnly: boolean; + transcript: string | null; +} + +export function scoreSegment(_features: SegmentFeatures, _weights: TeaserWeights): number { + return notImplemented("scoreSegment"); +} + +/** 0..1 speech-hook strength from a transcript excerpt (questions, exclamations, numbers, announcements). */ +export function speechHookScore(_text: string, _language?: string): number { + return notImplemented("speechHookScore"); +} + +export interface TeaserSelection { + clips: Array<{ segmentId: string; assetId: string; inMs: number; outMs: number }>; + hookSegmentId: string | null; + /** Why each clip was chosen, for plan rationale. */ + notes: string[]; +} + +/** + * Pick clips for a goal and profile: a hook inside hookWindowMs, rising + * tension, cut before the payoff (open loop), never paid-only segments, + * total within the profile's duration bounds. + */ +export function selectTeaserSegments(_input: { + segments: ScoredSegmentInput[]; + profile: PlatformProfile; + goal: MediaProjectGoal; + targetMs: number; + weights: TeaserWeights; + endCardMs: number; +}): TeaserSelection { + return notImplemented("selectTeaserSegments"); +} + +/** Regularized logistic regression of conversion on features; null below TEASER_LEARNING_MIN_SAMPLES. */ +export function fitTeaserWeights( + _samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>, + _prior: TeaserWeights, +): TeaserWeights | null { + return notImplemented("fitTeaserWeights"); +} + +/** Deterministic first-cut EDL for a project and profile; the agent refines it. */ +export function autoPlanEdl(_input: { + goal: MediaProjectGoal; + profile: PlatformProfile; + brandKit: BrandKit | null; + segments: ScoredSegmentInput[]; + weights: TeaserWeights; + assets: Map< + string, + { durationMs: number | null; width: number | null; height: number | null; hasSpeech: boolean } + >; + /** Person anonymization defaults to carry into redactions. */ + people: Array<{ id: string; anonymizeByDefault: string }>; + marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; + ctaText: string | null; +}): Edl { + return notImplemented("autoPlanEdl"); +} + +export function rankThumbnailFrames( + _candidates: Array<{ + tMs: number; + sharpness: number; + face: number; + thirds: number; + contrast: number; + }>, + _count: number, +): number[] { + return notImplemented("rankThumbnailFrames"); +} diff --git a/packages/media/tsconfig.json b/packages/media/tsconfig.json new file mode 100644 index 0000000..c699328 --- /dev/null +++ b/packages/media/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "noEmit": true + }, + "include": ["src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2acb6e..179bf2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,9 @@ importers: '@rakazo/logging': specifier: workspace:* version: link:../../packages/logging + '@rakazo/media': + specifier: workspace:* + version: link:../../packages/media '@rakazo/memory': specifier: workspace:* version: link:../../packages/memory @@ -176,6 +179,9 @@ importers: expo-localization: specifier: 57.0.2 version: 57.0.2(expo@57.0.26)(react@19.2.3) + expo-media-library: + specifier: 57.0.5 + version: 57.0.5(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3)) expo-modules-core: specifier: 57.0.20 version: 57.0.20(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3) @@ -203,6 +209,9 @@ importers: expo-updates: specifier: 57.0.24 version: 57.0.24(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3) + expo-video: + specifier: 57.0.5 + version: 57.0.5(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3) react: specifier: 19.2.3 version: 19.2.3 @@ -288,6 +297,9 @@ importers: '@rakazo/ui-web': specifier: workspace:* version: link:../../packages/ui-web + '@tanstack/react-virtual': + specifier: 3.14.13 + version: 3.14.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3) better-auth: specifier: 1.7.4 version: 1.7.4(@opentelemetry/api@1.9.1)(@prisma/client@7.10.0(prisma@7.10.0(@types/react-dom@19.2.7(@types/react@19.2.18))(@types/react@19.2.18)(magicast@0.5.4)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(typescript@7.0.2))(typescript@7.0.2))(mysql2@3.15.3)(pg@8.23.0)(prisma@7.10.0(@types/react-dom@19.2.7(@types/react@19.2.18))(@types/react@19.2.18)(magicast@0.5.4)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(typescript@7.0.2))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.5.0)(jsdom@30.0.1(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@26.5.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) @@ -579,6 +591,12 @@ importers: '@rakazo/logging': specifier: workspace:* version: link:../logging + '@rakazo/media': + specifier: workspace:* + version: link:../media + '@resvg/resvg-js': + specifier: 2.6.2 + version: 2.6.2 chat: specifier: 4.40.0 version: 4.40.0(zod@4.6.2) @@ -800,6 +818,22 @@ importers: specifier: 4.1.11 version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.5.0)(jsdom@30.0.1(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@26.5.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)) + packages/media: + dependencies: + '@rakazo/contracts': + specifier: workspace:* + version: link:../contracts + zod: + specifier: 4.6.2 + version: 4.6.2 + devDependencies: + typescript: + specifier: 7.0.2 + version: 7.0.2 + vitest: + specifier: 4.1.11 + version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.5.0)(jsdom@30.0.1(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@26.5.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)) + packages/memory: dependencies: '@rakazo/adapter-kit': @@ -3800,6 +3834,82 @@ packages: '@types/react': optional: true + '@resvg/resvg-js-android-arm-eabi@2.6.2': + resolution: {integrity: sha512-FrJibrAk6v29eabIPgcTUMPXiEz8ssrAk7TXxsiZzww9UTQ1Z5KAbFJs+Z0Ez+VZTYgnE5IQJqBcoSiMebtPHA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@resvg/resvg-js-android-arm64@2.6.2': + resolution: {integrity: sha512-VcOKezEhm2VqzXpcIJoITuvUS/fcjIw5NA/w3tjzWyzmvoCdd+QXIqy3FBGulWdClvp4g+IfUemigrkLThSjAQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@resvg/resvg-js-darwin-arm64@2.6.2': + resolution: {integrity: sha512-nmok2LnAd6nLUKI16aEB9ydMC6Lidiiq2m1nEBDR1LaaP7FGs4AJ90qDraxX+CWlVuRlvNjyYJTNv8qFjtL9+A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@resvg/resvg-js-darwin-x64@2.6.2': + resolution: {integrity: sha512-GInyZLjgWDfsVT6+SHxQVRwNzV0AuA1uqGsOAW+0th56J7Nh6bHHKXHBWzUrihxMetcFDmQMAX1tZ1fZDYSRsw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@resvg/resvg-js-linux-arm-gnueabihf@2.6.2': + resolution: {integrity: sha512-YIV3u/R9zJbpqTTNwTZM5/ocWetDKGsro0SWp70eGEM9eV2MerWyBRZnQIgzU3YBnSBQ1RcxRZvY/UxwESfZIw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@resvg/resvg-js-linux-arm64-gnu@2.6.2': + resolution: {integrity: sha512-zc2BlJSim7YR4FZDQ8OUoJg5holYzdiYMeobb9pJuGDidGL9KZUv7SbiD4E8oZogtYY42UZEap7dqkkYuA91pg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@resvg/resvg-js-linux-arm64-musl@2.6.2': + resolution: {integrity: sha512-3h3dLPWNgSsD4lQBJPb4f+kvdOSJHa5PjTYVsWHxLUzH4IFTJUAnmuWpw4KqyQ3NA5QCyhw4TWgxk3jRkQxEKg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@resvg/resvg-js-linux-x64-gnu@2.6.2': + resolution: {integrity: sha512-IVUe+ckIerA7xMZ50duAZzwf1U7khQe2E0QpUxu5MBJNao5RqC0zwV/Zm965vw6D3gGFUl7j4m+oJjubBVoftw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@resvg/resvg-js-linux-x64-musl@2.6.2': + resolution: {integrity: sha512-UOf83vqTzoYQO9SZ0fPl2ZIFtNIz/Rr/y+7X8XRX1ZnBYsQ/tTb+cj9TE+KHOdmlTFBxhYzVkP2lRByCzqi4jQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@resvg/resvg-js-win32-arm64-msvc@2.6.2': + resolution: {integrity: sha512-7C/RSgCa+7vqZ7qAbItfiaAWhyRSoD4l4BQAbVDqRRsRgY+S+hgS3in0Rxr7IorKUpGE69X48q6/nOAuTJQxeQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@resvg/resvg-js-win32-ia32-msvc@2.6.2': + resolution: {integrity: sha512-har4aPAlvjnLcil40AC77YDIk6loMawuJwFINEM7n0pZviwMkMvjb2W5ZirsNOZY4aDbo5tLx0wNMREp5Brk+w==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@resvg/resvg-js-win32-x64-msvc@2.6.2': + resolution: {integrity: sha512-ZXtYhtUr5SSaBrUDq7DiyjOFJqBVL/dOBN7N/qmi/pO0IgiWW/f/ue3nbvu9joWE5aAKDoIzy/CxsY0suwGosQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@resvg/resvg-js@2.6.2': + resolution: {integrity: sha512-xBaJish5OeGmniDj9cW5PRa/PtmuVU3ziqrbr5xJj901ZDN4TosrVaNZpEiLZAxdfnhAe7uQ7QFWfjPe9d9K2Q==} + engines: {node: '>= 10'} + '@rolldown/binding-android-arm-eabi@1.2.8': resolution: {integrity: sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4116,6 +4226,15 @@ packages: peerDependencies: vite: ^5.2.0 || ^6 || ^7 || ^8 + '@tanstack/react-virtual@3.14.13': + resolution: {integrity: sha512-JbDTAwtzZ99aOeCrAfW5EsE5KSq5RWh6Af2dtFwyLIIk48Ja7vm6n4axu/43T3vfjFnEGJalxJ1wyYjSQD6bSg==} + peerDependencies: + react: 19.2.3 + react-dom: 19.2.3 + + '@tanstack/virtual-core@3.17.11': + resolution: {integrity: sha512-+ILjvtHup6Y2hzQ6YzwMgX1Q+oQpxEGOXCEsCNaPoIP0VxMbizIBTmYTDtkerkIQS8/CbP1BRuyt8V/8BCsy1g==} + '@testcontainers/postgresql@12.1.0': resolution: {integrity: sha512-Pjf2VSVNirEPfz36nidyrVAnZvc2YhajOznY4VgyEsvfTd5qiMNOuPq96drREvxAUtXl5SFLX7vXj7sSq4aTcA==} @@ -6165,6 +6284,12 @@ packages: peerDependencies: expo: '*' + expo-media-library@57.0.5: + resolution: {integrity: sha512-amSLrLsS7Gl4nCqbc2PlVbHj5jK9kQLsSOQgJ2QasHCSHzYXw6Ui7aolxX5APg9P4TgYPE0yWn46ndifh52bRw==} + peerDependencies: + expo: '*' + react-native: '*' + expo-modules-autolinking@57.0.13: resolution: {integrity: sha512-Hj5NjZRlfccazhKab+wK19leRVUr1Y/PUERaRvegIbUSkq4GVxKq6lU9QHlKQyC16MdD/VS4iwsiLivB4o7lvQ==} hasBin: true @@ -6279,6 +6404,13 @@ packages: expo-dev-client: optional: true + expo-video@57.0.5: + resolution: {integrity: sha512-VK2o+d6rb+tVeL2DI6d0F58pRV8d8bctLpOrMJEHRYtMIK5iwvjjrRx2mciYN8/d04SgWC8TkI+wuZ8Sbp/yuA==} + peerDependencies: + expo: '*' + react: 19.2.3 + react-native: '*' + expo@57.0.26: resolution: {integrity: sha512-U1rsFwcuZNnU61Z/YLnnVC/ZqP0wpxh/rCKUMff3ot/rhLpxFY1zxEqu4vyaexyvBAB1mwpl9lA/ltcHBoQGyg==} hasBin: true @@ -13559,6 +13691,57 @@ snapshots: optionalDependencies: '@types/react': 19.2.18 + '@resvg/resvg-js-android-arm-eabi@2.6.2': + optional: true + + '@resvg/resvg-js-android-arm64@2.6.2': + optional: true + + '@resvg/resvg-js-darwin-arm64@2.6.2': + optional: true + + '@resvg/resvg-js-darwin-x64@2.6.2': + optional: true + + '@resvg/resvg-js-linux-arm-gnueabihf@2.6.2': + optional: true + + '@resvg/resvg-js-linux-arm64-gnu@2.6.2': + optional: true + + '@resvg/resvg-js-linux-arm64-musl@2.6.2': + optional: true + + '@resvg/resvg-js-linux-x64-gnu@2.6.2': + optional: true + + '@resvg/resvg-js-linux-x64-musl@2.6.2': + optional: true + + '@resvg/resvg-js-win32-arm64-msvc@2.6.2': + optional: true + + '@resvg/resvg-js-win32-ia32-msvc@2.6.2': + optional: true + + '@resvg/resvg-js-win32-x64-msvc@2.6.2': + optional: true + + '@resvg/resvg-js@2.6.2': + optionalDependencies: + '@resvg/resvg-js-android-arm-eabi': 2.6.2 + '@resvg/resvg-js-android-arm64': 2.6.2 + '@resvg/resvg-js-darwin-arm64': 2.6.2 + '@resvg/resvg-js-darwin-x64': 2.6.2 + '@resvg/resvg-js-linux-arm-gnueabihf': 2.6.2 + '@resvg/resvg-js-linux-arm64-gnu': 2.6.2 + '@resvg/resvg-js-linux-arm64-musl': 2.6.2 + '@resvg/resvg-js-linux-x64-gnu': 2.6.2 + '@resvg/resvg-js-linux-x64-musl': 2.6.2 + '@resvg/resvg-js-win32-arm64-msvc': 2.6.2 + '@resvg/resvg-js-win32-ia32-msvc': 2.6.2 + '@resvg/resvg-js-win32-x64-msvc': 2.6.2 + '@rolldown/binding-android-arm-eabi@1.2.8': optional: true @@ -13847,6 +14030,14 @@ snapshots: tailwindcss: 4.3.3 vite: 8.3.0(@types/node@26.5.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0) + '@tanstack/react-virtual@3.14.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': + dependencies: + '@tanstack/virtual-core': 3.17.11 + react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) + + '@tanstack/virtual-core@3.17.11': {} + '@testcontainers/postgresql@12.1.0': dependencies: testcontainers: 12.1.0 @@ -16233,6 +16424,11 @@ snapshots: expo: 57.0.26(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.16)(expo-router@57.0.24)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-webview@13.16.1(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-json-utils: 57.0.2 + expo-media-library@57.0.5(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3)): + dependencies: + expo: 57.0.26(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.16)(expo-router@57.0.24)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-webview@13.16.1(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3) + expo-modules-autolinking@57.0.13(typescript@6.0.3): dependencies: '@expo/require-utils': 57.0.5(typescript@6.0.3) @@ -16391,6 +16587,12 @@ snapshots: transitivePeerDependencies: - supports-color + expo-video@57.0.5(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3): + dependencies: + expo: 57.0.26(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.16)(expo-router@57.0.24)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-webview@13.16.1(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + react: 19.2.3 + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3) + expo@57.0.26(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.16)(expo-router@57.0.24)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-webview@13.16.1(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3)(typescript@6.0.3): dependencies: '@babel/runtime': 7.29.7 From 7ccc96ac111f1116082719a1f2799b5a1e061356 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 17:34:04 +0000 Subject: [PATCH 02/16] feat(media): shared geo, path-safety and profile loading helpers Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- packages/media/src/geo.ts | 63 ++++++++++--- packages/media/src/node.ts | 86 ++++++++++++++++-- packages/media/src/profiles.ts | 60 ++++++++++--- packages/media/src/shared.test.ts | 144 ++++++++++++++++++++++++++++++ 4 files changed, 322 insertions(+), 31 deletions(-) create mode 100644 packages/media/src/shared.test.ts diff --git a/packages/media/src/geo.ts b/packages/media/src/geo.ts index 2efc882..0cda7a6 100644 --- a/packages/media/src/geo.ts +++ b/packages/media/src/geo.ts @@ -1,28 +1,65 @@ -import { notImplemented } from "./not-implemented.js"; +const EARTH_RADIUS_KM = 6371.0088; + +function toRadians(degrees: number): number { + return (degrees * Math.PI) / 180; +} /** Great-circle distance in kilometres. */ export function haversineKm( - _a: { lat: number; lon: number }, - _b: { lat: number; lon: number }, + a: { lat: number; lon: number }, + b: { lat: number; lon: number }, ): number { - return notImplemented("haversineKm"); + const dLat = toRadians(b.lat - a.lat); + const dLon = toRadians(b.lon - a.lon); + const h = + Math.sin(dLat / 2) ** 2 + + Math.cos(toRadians(a.lat)) * Math.cos(toRadians(b.lat)) * Math.sin(dLon / 2) ** 2; + return 2 * EARTH_RADIUS_KM * Math.asin(Math.min(1, Math.sqrt(h))); } /** Round a coordinate to two decimals (about 1 km) before storage or display. */ -export function roundCoordinate(_value: number): number { - return notImplemented("roundCoordinate"); +export function roundCoordinate(value: number): number { + return Math.round(value * 100) / 100; } -/** Cosine similarity of two equal-length vectors. */ -export function cosineSimilarity(_a: ArrayLike, _b: ArrayLike): number { - return notImplemented("cosineSimilarity"); +/** Cosine similarity of two equal-length vectors; 0 when either is all zeros. */ +export function cosineSimilarity(a: ArrayLike, b: ArrayLike): number { + if (a.length !== b.length) throw new Error("vector length mismatch"); + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < a.length; i++) { + const x = a[i] as number; + const y = b[i] as number; + dot += x * y; + normA += x * x; + normB += y * y; + } + if (normA === 0 || normB === 0) return 0; + return dot / Math.sqrt(normA * normB); } /** Encode Float32 vectors as little-endian bytes for MediaAnalysis.vectors and centroids. */ -export function encodeVectors(_vectors: number[][]): Uint8Array { - return notImplemented("encodeVectors"); +export function encodeVectors(vectors: ArrayLike[]): Uint8Array { + const dim = vectors[0]?.length ?? 0; + const bytes = new Uint8Array(vectors.length * dim * 4); + const view = new DataView(bytes.buffer); + vectors.forEach((vector, row) => { + if (vector.length !== dim) throw new Error("vector length mismatch"); + for (let i = 0; i < dim; i++) view.setFloat32((row * dim + i) * 4, vector[i] as number, true); + }); + return bytes; } -export function decodeVectors(_bytes: Uint8Array, _dim: number): Float32Array[] { - return notImplemented("decodeVectors"); +export function decodeVectors(bytes: Uint8Array, dim: number): Float32Array[] { + if (dim <= 0 || bytes.byteLength % (dim * 4) !== 0) throw new Error("invalid vector bytes"); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const rows = bytes.byteLength / (dim * 4); + const out: Float32Array[] = []; + for (let row = 0; row < rows; row++) { + const vector = new Float32Array(dim); + for (let i = 0; i < dim; i++) vector[i] = view.getFloat32((row * dim + i) * 4, true); + out.push(vector); + } + return out; } diff --git a/packages/media/src/node.ts b/packages/media/src/node.ts index 5d6911a..d316369 100644 --- a/packages/media/src/node.ts +++ b/packages/media/src/node.ts @@ -1,18 +1,88 @@ +import { lstat, readdir, readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; import type { PlatformProfile } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; import type { ProfileLoadError } from "./profiles.js"; +import { parseProfiles } from "./profiles.js"; /** Directory of the profile JSON files shipped with this package. */ -export const BUILTIN_PROFILES_DIR = new URL("../profiles/", import.meta.url); +export const BUILTIN_PROFILES_DIR = fileURLToPath(new URL("../profiles/", import.meta.url)); -/** Read every *.json in a directory; missing directory yields no profiles. */ +/** Read every *.json in a directory; a missing directory yields no profiles. */ export async function loadProfilesFromDir( - _dir: string | URL, + dir: string, ): Promise<{ profiles: PlatformProfile[]; errors: ProfileLoadError[] }> { - return notImplemented("loadProfilesFromDir"); + let names: string[]; + try { + names = (await readdir(dir)).filter((name) => name.endsWith(".json")).sort(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return { profiles: [], errors: [] }; + throw error; + } + const documents: Array<{ file: string; json: unknown }> = []; + const errors: ProfileLoadError[] = []; + for (const name of names) { + const file = path.join(dir, name); + try { + documents.push({ file: name, json: JSON.parse(await readFile(file, "utf8")) }); + } catch (error) { + errors.push({ file: name, message: error instanceof Error ? error.message : String(error) }); + } + } + const parsed = parseProfiles(documents); + return { profiles: parsed.profiles, errors: [...errors, ...parsed.errors] }; } -/** Resolve a path inside root; rejects traversal and symlinks that escape it. */ -export async function resolveInside(_root: string, _relative: string): Promise { - return notImplemented("resolveInside"); +/** Expand a leading "~/" against the given home directory. */ +export function expandHome(input: string, home: string): string { + return input === "~" ? home : input.startsWith("~/") ? path.join(home, input.slice(2)) : input; +} + +export class PathOutsideRootError extends Error { + constructor(readonly requested: string) { + super("path is outside the allowed root"); + this.name = "PathOutsideRootError"; + } +} + +function isInside(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +/** + * Resolve `relative` inside `root`. Rejects absolute inputs, traversal, NUL + * bytes, and any existing path whose real location (after symlinks) leaves the + * real root. A path that does not exist yet is checked through its nearest + * existing ancestor. + */ +export async function resolveInside(root: string, relative: string): Promise { + if (relative.includes("\0") || path.isAbsolute(relative)) + throw new PathOutsideRootError(relative); + const realRoot = await realpath(root); + const lexical = path.resolve(realRoot, relative); + if (!isInside(realRoot, lexical)) throw new PathOutsideRootError(relative); + let probe = lexical; + for (;;) { + try { + await lstat(probe); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + const parent = path.dirname(probe); + if (parent === probe) throw new PathOutsideRootError(relative); + probe = parent; + } + } + const real = await realpath(probe); + if (!isInside(realRoot, real)) throw new PathOutsideRootError(relative); + return lexical; +} + +/** Like resolveInside for an absolute path that must lie within root. */ +export async function assertInside(root: string, absolute: string): Promise { + if (!path.isAbsolute(absolute)) throw new PathOutsideRootError(absolute); + const realRoot = await realpath(root); + const relative = path.relative(realRoot, path.resolve(absolute)); + return resolveInside(realRoot, relative === "" ? "." : relative); } diff --git a/packages/media/src/profiles.ts b/packages/media/src/profiles.ts index c78fe82..3cfb77d 100644 --- a/packages/media/src/profiles.ts +++ b/packages/media/src/profiles.ts @@ -1,5 +1,5 @@ import type { PlatformProfile, PlatformProfileSummary } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { PlatformProfileSchema } from "@rakazo/contracts"; export const PROFILE_STALE_DAYS = 90; @@ -14,25 +14,65 @@ export interface ProfileSet { } /** Parse raw JSON documents; invalid ones become errors and are skipped whole. */ -export function parseProfiles(_documents: Array<{ file: string; json: unknown }>): { +export function parseProfiles(documents: Array<{ file: string; json: unknown }>): { profiles: PlatformProfile[]; errors: ProfileLoadError[]; } { - return notImplemented("parseProfiles"); + const profiles: PlatformProfile[] = []; + const errors: ProfileLoadError[] = []; + const seen = new Set(); + for (const document of documents) { + const parsed = PlatformProfileSchema.safeParse(document.json); + if (!parsed.success) { + errors.push({ + file: document.file, + message: parsed.error.issues + .map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + .join("; "), + }); + continue; + } + if (seen.has(parsed.data.id)) { + errors.push({ file: document.file, message: `duplicate profile id ${parsed.data.id}` }); + continue; + } + seen.add(parsed.data.id); + profiles.push(parsed.data); + } + return { profiles, errors }; } /** Private profiles override builtin ones with the same id. */ export function mergeProfiles( - _builtin: PlatformProfile[], - _private: PlatformProfile[], + builtin: PlatformProfile[], + privateProfiles: PlatformProfile[], ): ProfileSet { - return notImplemented("mergeProfiles"); + const profiles: ProfileSet["profiles"] = new Map(); + for (const profile of builtin) profiles.set(profile.id, { profile, origin: "builtin" }); + for (const profile of privateProfiles) profiles.set(profile.id, { profile, origin: "private" }); + return { profiles, errors: [] }; +} + +export function isProfileStale(profile: PlatformProfile, now: Date): boolean { + const reviewed = Date.parse(`${profile.reviewedAt}T00:00:00Z`); + return now.getTime() - reviewed > PROFILE_STALE_DAYS * 24 * 60 * 60 * 1000; } export function summarizeProfile( - _profile: PlatformProfile, - _origin: "builtin" | "private", - _now: Date, + profile: PlatformProfile, + origin: "builtin" | "private", + now: Date, ): PlatformProfileSummary { - return notImplemented("summarizeProfile"); + return { + id: profile.id, + name: profile.name, + kind: profile.kind, + origin, + unverified: profile.unverified, + stale: isProfileStale(profile, now), + aspectRatios: profile.video.aspectRatios, + hardMinMs: profile.duration.hardMinMs, + hardMaxMs: profile.duration.hardMaxMs, + recommendedMs: profile.duration.recommended, + }; } diff --git a/packages/media/src/shared.test.ts b/packages/media/src/shared.test.ts new file mode 100644 index 0000000..00ff282 --- /dev/null +++ b/packages/media/src/shared.test.ts @@ -0,0 +1,144 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + cosineSimilarity, + decodeVectors, + encodeVectors, + haversineKm, + roundCoordinate, +} from "./geo.js"; +import { + assertInside, + expandHome, + loadProfilesFromDir, + PathOutsideRootError, + resolveInside, +} from "./node.js"; +import { mergeProfiles, parseProfiles, summarizeProfile } from "./profiles.js"; + +const profileJson = (id: string, overrides: Record = {}) => ({ + id, + name: `Profile ${id}`, + kind: "free_social", + reviewedAt: "2026-09-01", + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 500, + }, + duration: { hardMinMs: 1000, hardMaxMs: 60000, recommended: [15000] }, + safeZones: { topPct: 10, bottomPct: 20, leftPct: 5, rightPct: 10 }, + captions: { recommended: true }, + audio: {}, + text: { captionMaxChars: 2200, hashtagLimit: 5, linkPolicy: "bio_only" }, + content: { thresholds: { explicit_nudity: 0.2 } }, + disclosure: { aiAlteredLabelRequired: true }, + consent: { requirePerformerConsent: false, requireAgeVerification: false }, + export: {}, + ...overrides, +}); + +describe("geo", () => { + it("measures distance and rounds coordinates", () => { + expect(haversineKm({ lat: 52.52, lon: 13.405 }, { lat: 48.137, lon: 11.575 })).toBeCloseTo( + 504, + -1, + ); + expect(haversineKm({ lat: 1, lon: 1 }, { lat: 1, lon: 1 })).toBe(0); + expect(roundCoordinate(52.51678)).toBe(52.52); + }); + + it("computes cosine similarity and round-trips vectors", () => { + expect(cosineSimilarity([1, 0], [1, 0])).toBe(1); + expect(cosineSimilarity([1, 0], [0, 1])).toBe(0); + expect(cosineSimilarity([0, 0], [1, 1])).toBe(0); + expect(() => cosineSimilarity([1], [1, 2])).toThrow(); + const decoded = decodeVectors( + encodeVectors([ + [1, 2.5], + [-3, 0], + ]), + 2, + ); + expect(Array.from(decoded[1] ?? [])).toEqual([-3, 0]); + expect(() => decodeVectors(new Uint8Array(6), 2)).toThrow(); + }); +}); + +describe("profiles", () => { + it("skips invalid and duplicate documents whole", () => { + const result = parseProfiles([ + { file: "a.json", json: profileJson("a") }, + { file: "b.json", json: { id: "b" } }, + { file: "a2.json", json: profileJson("a") }, + { file: "c.json", json: profileJson("c", { duration: { hardMinMs: 9, hardMaxMs: 1 } }) }, + ]); + expect(result.profiles.map((p) => p.id)).toEqual(["a"]); + expect(result.errors.map((e) => e.file)).toEqual(["b.json", "a2.json", "c.json"]); + }); + + it("lets private profiles override builtin ones and flags stale ones", () => { + const builtin = parseProfiles([{ file: "x", json: profileJson("shared") }]).profiles; + const own = parseProfiles([ + { file: "y", json: profileJson("shared", { name: "Mine" }) }, + ]).profiles; + const merged = mergeProfiles(builtin, own); + expect(merged.profiles.get("shared")?.origin).toBe("private"); + expect(merged.profiles.get("shared")?.profile.name).toBe("Mine"); + const summary = summarizeProfile(builtin[0]!, "builtin", new Date("2027-01-15T00:00:00Z")); + expect(summary.stale).toBe(true); + expect(summarizeProfile(builtin[0]!, "builtin", new Date("2026-09-20T00:00:00Z")).stale).toBe( + false, + ); + }); +}); + +describe("node helpers", () => { + let root: string; + let outside: string; + + beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "media-root-")); + outside = await mkdtemp(path.join(tmpdir(), "media-outside-")); + await mkdir(path.join(root, "a")); + await writeFile(path.join(root, "a", "f.txt"), "x"); + await symlink(outside, path.join(root, "escape")); + await writeFile(path.join(root, "good.json"), JSON.stringify(profileJson("good"))); + await writeFile(path.join(root, "broken.json"), "{"); + }); + + afterAll(async () => { + await rm(root, { recursive: true, force: true }); + await rm(outside, { recursive: true, force: true }); + }); + + it("resolves paths inside the root and rejects escapes", async () => { + expect(await resolveInside(root, "a/f.txt")).toMatch(/a\/f\.txt$/); + expect(await resolveInside(root, "a/new/file.mp4")).toMatch(/a\/new\/file\.mp4$/); + await expect(resolveInside(root, "../x")).rejects.toBeInstanceOf(PathOutsideRootError); + await expect(resolveInside(root, "/etc/passwd")).rejects.toBeInstanceOf(PathOutsideRootError); + await expect(resolveInside(root, "escape/secret")).rejects.toBeInstanceOf(PathOutsideRootError); + await expect(resolveInside(root, "a/\0x")).rejects.toBeInstanceOf(PathOutsideRootError); + await expect(assertInside(root, path.join(outside, "x"))).rejects.toBeInstanceOf( + PathOutsideRootError, + ); + expect(await assertInside(root, path.join(root, "a"))).toMatch(/\/a$/); + }); + + it("loads profile directories and reports broken files", async () => { + const loaded = await loadProfilesFromDir(root); + expect(loaded.profiles.map((p) => p.id)).toEqual(["good"]); + expect(loaded.errors.map((e) => e.file)).toEqual(["broken.json"]); + expect(await loadProfilesFromDir(path.join(root, "missing"))).toEqual({ + profiles: [], + errors: [], + }); + expect(expandHome("~/x", "/home/u")).toBe("/home/u/x"); + }); +}); From b5cef21f44c3f092381d770fe9b7e2a0b77972ce Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 17:47:24 +0000 Subject: [PATCH 03/16] feat(media): resvg motion graphics renderer with brand-kit templates Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- .../adapters/src/media-motion-renderer.ts | 10 + packages/adapters/src/media-motion/easing.ts | 56 +++ packages/adapters/src/media-motion/index.ts | 308 ++++++++++++ .../src/media-motion/media-motion.test.ts | 441 ++++++++++++++++++ .../adapters/src/media-motion/template.ts | 305 ++++++++++++ .../src/media-motion/templates/captions.ts | 182 ++++++++ .../src/media-motion/templates/end-cards.ts | 153 ++++++ .../src/media-motion/templates/intros.ts | 309 ++++++++++++ .../media-motion/templates/lower-thirds.ts | 245 ++++++++++ .../src/media-motion/templates/outros.ts | 106 +++++ 10 files changed, 2115 insertions(+) create mode 100644 packages/adapters/src/media-motion-renderer.ts create mode 100644 packages/adapters/src/media-motion/easing.ts create mode 100644 packages/adapters/src/media-motion/index.ts create mode 100644 packages/adapters/src/media-motion/media-motion.test.ts create mode 100644 packages/adapters/src/media-motion/template.ts create mode 100644 packages/adapters/src/media-motion/templates/captions.ts create mode 100644 packages/adapters/src/media-motion/templates/end-cards.ts create mode 100644 packages/adapters/src/media-motion/templates/intros.ts create mode 100644 packages/adapters/src/media-motion/templates/lower-thirds.ts create mode 100644 packages/adapters/src/media-motion/templates/outros.ts diff --git a/packages/adapters/src/media-motion-renderer.ts b/packages/adapters/src/media-motion-renderer.ts new file mode 100644 index 0000000..f43b1d9 --- /dev/null +++ b/packages/adapters/src/media-motion-renderer.ts @@ -0,0 +1,10 @@ +export type { + MotionFrameContext, + MotionTemplate, + ResvgMotionGraphicsRendererOptions, +} from "./media-motion/index.js"; +export { + MOTION_TEMPLATES, + MotionTemplateError, + ResvgMotionGraphicsRenderer, +} from "./media-motion/index.js"; diff --git a/packages/adapters/src/media-motion/easing.ts b/packages/adapters/src/media-motion/easing.ts new file mode 100644 index 0000000..95c42bc --- /dev/null +++ b/packages/adapters/src/media-motion/easing.ts @@ -0,0 +1,56 @@ +/** Easing and timing helpers shared by every motion template. All pure. */ + +export function clamp(value: number, min = 0, max = 1): number { + return Math.min(max, Math.max(min, value)); +} + +export function lerp(from: number, to: number, amount: number): number { + return from + (to - from) * amount; +} + +/** Linear progress 0..1 of `t` through the window [start, end]. */ +export function progress(t: number, start: number, end: number): number { + if (end <= start) return t >= end ? 1 : 0; + return clamp((t - start) / (end - start)); +} + +export function easeInCubic(x: number): number { + return x * x * x; +} + +export function easeOutCubic(x: number): number { + return 1 - (1 - x) ** 3; +} + +export function easeInOutCubic(x: number): number { + return x < 0.5 ? 4 * x * x * x : 1 - (-2 * x + 2) ** 3 / 2; +} + +export function easeOutExpo(x: number): number { + return x >= 1 ? 1 : 1 - 2 ** (-10 * x); +} + +/** Overshoots slightly before settling; for pop-in reveals. */ +export function easeOutBack(x: number): number { + const c1 = 1.70158; + const c3 = c1 + 1; + return 1 + c3 * (x - 1) ** 3 + c1 * (x - 1) ** 2; +} + +/** + * Opacity envelope for a clip of `durationMs`: fades in over `inMs` and out + * over the last `outMs`. + */ +export function fadeEnvelope(t: number, durationMs: number, inMs: number, outMs: number): number { + const fadeIn = inMs > 0 ? easeOutCubic(progress(t, 0, inMs)) : 1; + const fadeOut = outMs > 0 ? 1 - easeInCubic(progress(t, durationMs - outMs, durationMs)) : 1; + return clamp(Math.min(fadeIn, fadeOut)); +} + +/** Deterministic pseudo-random value in [0, 1) for integer seeds. */ +export function hash01(a: number, b = 0): number { + let h = Math.imul(a ^ 0x9e3779b9, 0x85ebca6b) ^ Math.imul(b + 0x632be5ab, 0xc2b2ae35); + h = Math.imul(h ^ (h >>> 15), 0x2c1b3c6d); + h = Math.imul(h ^ (h >>> 12), 0x297a2d39); + return ((h ^ (h >>> 15)) >>> 0) / 4294967296; +} diff --git a/packages/adapters/src/media-motion/index.ts b/packages/adapters/src/media-motion/index.ts new file mode 100644 index 0000000..1161c6b --- /dev/null +++ b/packages/adapters/src/media-motion/index.ts @@ -0,0 +1,308 @@ +/* + * Motion graphics renderer for Media Studio (intros, outros, lower thirds, + * caption previews, end cards). + * + * Technology decision: no Remotion and no headless Chromium. Remotion needs a + * paid company license above a team size and Chromium is a heavy, hard-to- + * sandbox runtime. Instead every template is a pure TypeScript function that + * returns an SVG document for a point in time. Frames are rasterized with + * @resvg/resvg-js (MPL-2.0; resvg itself is MPL-2.0, used unmodified as a + * library, commercially usable) and piped as PNGs into ffmpeg (LGPL/GPL + * depending on the build, invoked as a separate process with an argv array), + * which encodes a QuickTime Animation (qtrle, argb) MOV with alpha. Nothing + * here needs a network or a hosted service. + */ +import type { ChildProcess } from "node:child_process"; +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { readFile, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { MotionGraphicsRenderer, MotionRenderRequest } from "@rakazo/adapter-kit"; +import type { BrandKit, MotionTemplateInfo, MotionTemplateRef } from "@rakazo/contracts"; +import { BrandKitSchema, MotionTemplateRefSchema } from "@rakazo/contracts"; +import { assertInside } from "@rakazo/media/node"; +import type { RenderedImage, ResvgRenderOptions } from "@resvg/resvg-js"; +import { renderAsync } from "@resvg/resvg-js"; +import type { MotionFrameContext, MotionTemplate } from "./template.js"; +import { brandFonts, MotionTemplateError, validateDuration, validateParams } from "./template.js"; +import { captionBoldSocial, captionClean, captionKaraoke } from "./templates/captions.js"; +import { endCardCta, endCardMinimal } from "./templates/end-cards.js"; +import { + introElegantFade, + introGlitch, + introKineticType, + introLogoReveal, +} from "./templates/intros.js"; +import { lowerThirdBar, lowerThirdClean, lowerThirdPill } from "./templates/lower-thirds.js"; +import { outroLogo, outroSubscribe } from "./templates/outros.js"; + +export type { MotionFrameContext, MotionTemplate } from "./template.js"; +export { MotionTemplateError } from "./template.js"; + +export const MOTION_TEMPLATES: readonly MotionTemplate[] = [ + introLogoReveal, + introKineticType, + introGlitch, + introElegantFade, + outroLogo, + outroSubscribe, + lowerThirdClean, + lowerThirdBar, + lowerThirdPill, + captionClean, + captionKaraoke, + captionBoldSocial, + endCardCta, + endCardMinimal, +]; + +export function motionTemplate(id: string): MotionTemplate { + const template = MOTION_TEMPLATES.find((entry) => entry.info.id === id); + if (!template) throw new MotionTemplateError("unknown_template", `unknown template "${id}"`); + return template; +} + +export const MIN_FRAME_SIDE = 16; +export const MAX_FRAME_SIDE = 3840; +export const MAX_FPS = 60; +/** Default cap on width × height × frames: ten seconds of 1080p60. */ +export const DEFAULT_MAX_PIXEL_FRAMES = 1920 * 1080 * 60 * 10; +const MAX_LOGO_BYTES = 5 * 1024 * 1024; +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const FONT_EXTENSIONS = new Set([".ttf", ".otf", ".ttc"]); + +/** + * Read a logo as a data URI. Only PNG and SVG are accepted, and an SVG may not + * declare entities or reference anything outside itself. + */ +export async function loadLogoDataUri(file: string): Promise { + const extension = path.extname(file).toLowerCase(); + if (extension !== ".png" && extension !== ".svg") + throw new MotionTemplateError("invalid_asset", "logos must be PNG or SVG"); + const info = await stat(file); + if (!info.isFile() || info.size > MAX_LOGO_BYTES) + throw new MotionTemplateError("invalid_asset", "logo is not a file or is too large"); + const bytes = await readFile(file); + if (extension === ".png") { + if (!bytes.subarray(0, 8).equals(PNG_SIGNATURE)) + throw new MotionTemplateError("invalid_asset", "logo is not a PNG"); + return `data:image/png;base64,${bytes.toString("base64")}`; + } + const text = bytes.toString("utf8"); + if (!/]/i.test(text)) + throw new MotionTemplateError("invalid_asset", "logo is not an SVG"); + if (/ MAX_FRAME_SIDE) + throw new MotionTemplateError( + "invalid_size", + `frame sides must be ${MIN_FRAME_SIDE}..${MAX_FRAME_SIDE} px`, + ); + } + return { + template, + ctx: { + width: input.width, + height: input.height, + durationMs: ref.durationMs, + kit, + params: validateParams(template.info, ref.params), + logos: input.logos, + fonts: brandFonts(kit), + }, + }; +} + +export interface RasterizeOptions { + fontFiles: string[]; + defaultFontFamily?: string | null; + /** Fall back to system fonts for missing families and glyphs. */ + loadSystemFonts: boolean; +} + +export function rasterizeFrame(svg: string, options: RasterizeOptions): Promise { + const resvgOptions: ResvgRenderOptions = { + font: { + loadSystemFonts: options.loadSystemFonts, + fontFiles: options.fontFiles, + ...(options.defaultFontFamily ? { defaultFontFamily: options.defaultFontFamily } : {}), + }, + logLevel: "off", + }; + return renderAsync(svg, resvgOptions); +} + +export function frameCount(durationMs: number, fps: number): number { + return Math.max(1, Math.round((durationMs * fps) / 1000)); +} + +function abortError(): Error { + const error = new Error("motion render aborted"); + error.name = "AbortError"; + return error; +} + +export interface ResvgMotionGraphicsRendererOptions { + ffmpegPath?: string; + /** When set, every asset path and the output path must lie inside it. */ + root?: string; + maxPixelFrames?: number; +} + +export class ResvgMotionGraphicsRenderer implements MotionGraphicsRenderer { + readonly id = "resvg"; + private readonly ffmpegPath: string; + private readonly maxPixelFrames: number; + + constructor(private readonly options: ResvgMotionGraphicsRendererOptions = {}) { + this.ffmpegPath = options.ffmpegPath ?? "ffmpeg"; + this.maxPixelFrames = options.maxPixelFrames ?? DEFAULT_MAX_PIXEL_FRAMES; + } + + templates(): MotionTemplateInfo[] { + return MOTION_TEMPLATES.map((template) => structuredClone(template.info)); + } + + async render(request: MotionRenderRequest): Promise<{ path: string; hasAlpha: boolean }> { + const { signal } = request; + if (signal?.aborted) throw abortError(); + if (!Number.isFinite(request.fps) || request.fps < 1 || request.fps > MAX_FPS) + throw new MotionTemplateError("invalid_size", `fps must be 1..${MAX_FPS}`); + if (!path.isAbsolute(request.outPath) || path.extname(request.outPath).toLowerCase() !== ".mov") + throw new MotionTemplateError("invalid_asset", "outPath must be an absolute .mov path"); + + const outPath = await this.checkedPath(request.outPath); + const fontFiles: string[] = []; + for (const font of request.assets.fonts) { + const file = await this.checkedPath(font.path); + if (!FONT_EXTENSIONS.has(path.extname(file).toLowerCase()) || !(await stat(file)).isFile()) + throw new MotionTemplateError("invalid_asset", "fonts must be TTF, OTF, or TTC files"); + fontFiles.push(file); + } + const logos: string[] = []; + for (const logo of request.assets.logoPaths) { + logos.push(await loadLogoDataUri(await this.checkedPath(logo))); + } + + const { template, ctx } = buildFrameContext({ + template: request.template, + kit: request.brandKit, + width: request.width, + height: request.height, + logos, + }); + const frames = frameCount(ctx.durationMs, request.fps); + if (ctx.width * ctx.height * frames > this.maxPixelFrames) + throw new MotionTemplateError("invalid_size", "motion clip exceeds the pixel budget"); + + const raster: RasterizeOptions = { + fontFiles, + defaultFontFamily: ctx.fonts.body, + loadSystemFonts: true, + }; + const partial = path.join( + path.dirname(outPath), + `.${path.basename(outPath, ".mov")}.${randomUUID()}.partial.mov`, + ); + const child = spawn( + this.ffmpegPath, + [ + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-f", + "image2pipe", + "-framerate", + String(request.fps), + "-c:v", + "png", + "-i", + "pipe:0", + "-an", + "-c:v", + "qtrle", + "-pix_fmt", + "argb", + "-f", + "mov", + partial, + ], + { stdio: ["pipe", "ignore", "pipe"], shell: false }, + ); + const exited = waitForExit(child); + const kill = () => child.kill("SIGKILL"); + signal?.addEventListener("abort", kill, { once: true }); + try { + const stdin = child.stdin!; + stdin.on("error", () => {}); + for (let index = 0; index < frames; index += 1) { + if (signal?.aborted) throw abortError(); + const svg = template.frame((index * 1000) / request.fps, ctx); + const png = (await rasterizeFrame(svg, raster)).asPng(); + if (signal?.aborted) throw abortError(); + if (!stdin.write(png)) { + const result = await Promise.race([once(stdin, "drain").then(() => null), exited]); + if (result) throw ffmpegError(result); + } + } + stdin.end(); + const result = await exited; + if (signal?.aborted) throw abortError(); + if (result.code !== 0) throw ffmpegError(result); + await rename(partial, outPath); + return { path: outPath, hasAlpha: true }; + } catch (error) { + kill(); + await exited; + await rm(partial, { force: true }); + throw signal?.aborted ? abortError() : error; + } finally { + signal?.removeEventListener("abort", kill); + } + } + + private async checkedPath(file: string): Promise { + if (!path.isAbsolute(file) || file.includes("\0")) + throw new MotionTemplateError("invalid_asset", "asset paths must be absolute"); + return this.options.root ? assertInside(this.options.root, file) : path.resolve(file); + } +} + +interface ExitResult { + code: number | null; + stderr: string; +} + +function waitForExit(child: ChildProcess): Promise { + let stderr = ""; + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => { + stderr = (stderr + chunk).slice(-4000); + }); + return new Promise((resolve) => { + child.once("error", (error) => resolve({ code: -1, stderr: error.message })); + child.once("close", (code) => resolve({ code, stderr })); + }); +} + +function ffmpegError(result: ExitResult): Error { + return new Error(`ffmpeg failed (${result.code}): ${result.stderr.trim()}`); +} diff --git a/packages/adapters/src/media-motion/media-motion.test.ts b/packages/adapters/src/media-motion/media-motion.test.ts new file mode 100644 index 0000000..edb530a --- /dev/null +++ b/packages/adapters/src/media-motion/media-motion.test.ts @@ -0,0 +1,441 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { BrandKit, MotionTemplateRef } from "@rakazo/contracts"; +import { MotionTemplateInfoSchema } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { MotionTemplate } from "./index.js"; +import { + buildFrameContext, + loadLogoDataUri, + MOTION_TEMPLATES, + MotionTemplateError, + motionTemplate, + ResvgMotionGraphicsRenderer, + rasterizeFrame, +} from "./index.js"; +import { GLITCH_BURST_INTERVAL_MS } from "./templates/intros.js"; + +const DEJAVU_DIR = "/usr/share/fonts/truetype/dejavu"; +const DEJAVU = ["DejaVuSans.ttf", "DejaVuSans-Bold.ttf", "DejaVuSerif.ttf"].map((name) => + path.join(DEJAVU_DIR, name), +); +const hasDejaVu = DEJAVU.every((file) => existsSync(file)); +const hasFfmpeg = (() => { + try { + execFileSync("ffmpeg", ["-hide_banner", "-encoders"], { stdio: "pipe" }); + execFileSync("ffprobe", ["-version"], { stdio: "pipe" }); + return true; + } catch { + return false; + } +})(); + +const LOGO_SVG = + '' + + '' + + ''; + +const kit: BrandKit = { + name: "Test Brand", + colors: { + primary: "#2255cc", + secondary: "#9999aa", + accent: "#ff8800", + text: "#ffffff", + background: "#101820", + }, + fonts: [{ family: "DejaVu Sans", artifactId: "font-1", role: "heading" }], + logoArtifactIds: ["logo-1"], + captionPresets: [], + musicBeds: [], +}; + +/** Parameters that exercise every template with all its text slots filled. */ +const SAMPLE_PARAMS: Record = { + intro_logo_reveal: { tagline: "Make it simple" }, + intro_kinetic_type: { title: "Bold ideas move fast", tagline: "Since 2026" }, + intro_glitch: { tagline: "Energy" }, + intro_elegant_fade: { tagline: "Crafted slowly" }, + outro_logo: { message: "Thanks" }, + outro_subscribe: { cta: "Subscribe", handle: "@example" }, + lt_clean: { title: "Alex Example", subtitle: "Host" }, + lt_bar: { title: "Alex Example", subtitle: "Host", barColor: "accent" }, + lt_pill: { title: "Alex Example", subtitle: "Host" }, + caption_clean: { text: "A caption in the clean style" }, + caption_karaoke: { text: "A caption in the karaoke style" }, + caption_bold_social: { text: "Bold social caption" }, + endcard_cta: { headline: "Get the full video", cta: "Watch now", price: "Only 9 today" }, + endcard_minimal: { url: "example.com" }, +}; + +let dir: string; +let logoPath: string; +let logo: string; + +beforeAll(async () => { + dir = await mkdtemp(path.join(tmpdir(), "rakazo-motion-")); + logoPath = path.join(dir, "logo.svg"); + await writeFile(logoPath, LOGO_SVG); + logo = await loadLogoDataUri(logoPath); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +function context( + template: MotionTemplate, + overrides: { + params?: MotionTemplateRef["params"]; + durationMs?: number; + size?: [number, number]; + } = {}, +) { + const [width, height] = overrides.size ?? [640, 360]; + return buildFrameContext({ + template: { + templateId: template.info.id, + params: overrides.params ?? SAMPLE_PARAMS[template.info.id] ?? {}, + durationMs: overrides.durationMs ?? template.info.minDurationMs, + }, + kit, + width, + height, + logos: [logo], + }).ctx; +} + +const raster = { fontFiles: DEJAVU, defaultFontFamily: "DejaVu Sans", loadSystemFonts: false }; + +/** Mean relative luminance of an RGBA frame composited over a solid backdrop. */ +function meanLuminance(pixels: Buffer, backdrop: 0 | 1): number { + let sum = 0; + const count = pixels.length / 4; + for (let index = 0; index < pixels.length; index += 4) { + const alpha = pixels[index + 3]! / 255; + const channel = (offset: number) => { + const value = (pixels[index + offset]! / 255) * alpha + backdrop * (1 - alpha); + return value <= 0.03928 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4; + }; + sum += 0.2126 * channel(0) + 0.7152 * channel(1) + 0.0722 * channel(2); + } + return sum / count; +} + +/** + * Count brightness reversals of at least 10% relative luminance (the WCAG 2.3.1 + * flash threshold) per one-second window and return the worst window. A flash + * is a pair of opposing changes, so a steady fade counts as none; counting each + * reversal on its own is stricter than WCAG's three flashes per second. + */ +function maxReversalsPerSecond(series: number[], fps: number): number { + const transitions: number[] = []; + let direction = 0; + let extreme = series[0] ?? 0; + series.forEach((value, index) => { + const delta = value - extreme; + if (direction !== 0 && Math.sign(delta) === direction) { + extreme = value; + } else if (Math.abs(delta) >= 0.1) { + if (direction !== 0) transitions.push(index / fps); + direction = Math.sign(delta); + extreme = value; + } + }); + let worst = 0; + for (const start of transitions) { + worst = Math.max(worst, transitions.filter((at) => at >= start && at < start + 1).length); + } + return worst; +} + +describe("motion templates", () => { + it("ships the required set with valid, unique descriptors", () => { + const ids = MOTION_TEMPLATES.map((template) => template.info.id); + expect(new Set(ids).size).toBe(ids.length); + for (const template of MOTION_TEMPLATES) MotionTemplateInfoSchema.parse(template.info); + const count = (kind: string) => + MOTION_TEMPLATES.filter((template) => template.info.kind === kind).length; + expect([ + count("intro"), + count("outro"), + count("lower_third"), + count("caption"), + count("end_card"), + ]).toEqual([4, 2, 3, 3, 2]); + }); + + it.each(MOTION_TEMPLATES.map((template) => [template.info.id, template] as const))( + "%s renders its first, middle and last frame", + async (_id, template) => { + const ctx = context(template); + for (const t of [0, ctx.durationMs / 2, ctx.durationMs]) { + const svg = template.frame(t, ctx); + expect(svg.startsWith(" { + const allowed = new Set(Object.values(kit.colors).map((color) => color.toLowerCase())); + for (const template of MOTION_TEMPLATES) { + const ctx = context(template); + const svg = template.frame(ctx.durationMs / 2, ctx); + const colors = [...svg.matchAll(/(?:fill|stroke)="([^"]+)"/g)].map((match) => match[1]!); + for (const color of colors) { + if (color === "none" || color.startsWith("url(")) continue; + expect(allowed.has(color.toLowerCase()), `${template.info.id} uses ${color}`).toBe(true); + } + } + }); + + it.skipIf(!hasDejaVu)("renders deterministic middle frames", async () => { + const hashes: Record = {}; + for (const template of MOTION_TEMPLATES) { + const ctx = context(template); + const image = await rasterizeFrame(template.frame(ctx.durationMs / 2, ctx), raster); + hashes[template.info.id] = createHash("sha256") + .update(image.pixels) + .digest("hex") + .slice(0, 16); + } + expect(hashes).toMatchInlineSnapshot(` + { + "caption_bold_social": "4a132766b59cdf12", + "caption_clean": "6194d5224d8e97ec", + "caption_karaoke": "f2db9bc313e4ccea", + "endcard_cta": "943fa754c9a30803", + "endcard_minimal": "be2b9f833895bc36", + "intro_elegant_fade": "8f8a98930994243f", + "intro_glitch": "35074c7ad30d26bb", + "intro_kinetic_type": "52129b6c4f5581e3", + "intro_logo_reveal": "2baf101db044941b", + "lt_bar": "a13184996d2ffc2e", + "lt_clean": "ca6a4239283e111e", + "lt_pill": "30a7342e49735d69", + "outro_logo": "84b1cd23213f8a2e", + "outro_subscribe": "18733abc1b0f1296", + } + `); + }); + + it("never changes full-frame brightness more than three times a second in the glitch intro", async () => { + const template = motionTemplate("intro_glitch"); + const ctx = context(template, { durationMs: 4000, size: [160, 90] }); + const fps = 25; + const dark: number[] = []; + const light: number[] = []; + for (let index = 0; index <= (ctx.durationMs * fps) / 1000; index += 1) { + const image = await rasterizeFrame(template.frame((index * 1000) / fps, ctx), { + fontFiles: [], + loadSystemFonts: true, + }); + dark.push(meanLuminance(image.pixels, 0)); + light.push(meanLuminance(image.pixels, 1)); + } + expect(1000 / GLITCH_BURST_INTERVAL_MS).toBeLessThanOrEqual(3); + expect(maxReversalsPerSecond(dark, fps)).toBeLessThanOrEqual(3); + expect(maxReversalsPerSecond(light, fps)).toBeLessThanOrEqual(3); + }); + + it("detects a frame that does flash too fast", () => { + const flashing = Array.from({ length: 50 }, (_, index) => (index % 5 < 2 ? 0.9 : 0.05)); + expect(maxReversalsPerSecond(flashing, 50)).toBeGreaterThan(3); + }); +}); + +describe("motion params and escaping", () => { + const lowerThird = motionTemplate("lt_clean"); + + it.each([ + [{ title: "x", extra: "y" }, "unknown_param"], + [{ title: 3 }, "invalid_param"], + [{ title: "a\u0007b" }, "invalid_param"], + [{ subtitle: "only" }, "missing_param"], + ] as const)("rejects %j", (params, code) => { + expect(() => context(lowerThird, { params })).toThrow( + expect.objectContaining({ code }) as unknown as Error, + ); + }); + + it("rejects oversized strings, unknown colors, durations, and frame sizes", () => { + expect(() => context(lowerThird, { params: { title: "x".repeat(201) } })).toThrow(); + expect(() => + context(motionTemplate("lt_bar"), { params: { title: "x", barColor: "#ff0000" } }), + ).toThrow(MotionTemplateError); + expect(() => context(lowerThird, { durationMs: 60_000 })).toThrow(MotionTemplateError); + expect(() => context(lowerThird, { size: [8000, 360] })).toThrow(MotionTemplateError); + expect(() => + buildFrameContext({ + template: { templateId: "nope", params: {}, durationMs: 2000 }, + kit, + width: 640, + height: 360, + logos: [], + }), + ).toThrow(MotionTemplateError); + }); + + it("escapes text and keeps hostile input out of the markup", async () => { + const hostile = ` & 'quotes' ]]> + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-library-validation + + + com.apple.security.personal-information.photos-library + + + diff --git a/apps/desktop/src/media-ipc.test.ts b/apps/desktop/src/media-ipc.test.ts new file mode 100644 index 0000000..f1d0bf7 --- /dev/null +++ b/apps/desktop/src/media-ipc.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it, vi } from "vitest"; +import type { MediaIpcEvent, MediaIpcOptions } from "./media-ipc.js"; +import { + MEDIA_PHOTOS_CHANNELS, + outcomeForError, + parseSyncRequest, + registerMediaIpc, +} from "./media-ipc.js"; +import type { PhotosHelper } from "./media-photos-helper.js"; +import { PhotosHelperError } from "./media-photos-helper.js"; +import type { PhotosSyncResult } from "./media-photos-sync.js"; +import { PhotosSyncError } from "./media-photos-sync.js"; + +type Listener = (event: MediaIpcEvent, ...args: unknown[]) => unknown; + +const result: PhotosSyncResult = { + indexed: 2, + created: 2, + updated: 0, + unchanged: 0, + uploaded: 0, + fullListing: true, +}; + +function setup(overrides: Partial = {}) { + const handlers = new Map(); + const helper = { + authorize: vi.fn(async () => "authorized" as const), + } as unknown as PhotosHelper; + const options: MediaIpcOptions = { + ipcMain: { handle: (channel, listener) => handlers.set(channel, listener) }, + platform: "darwin", + helper, + isTrusted: () => true, + runSync: vi.fn(async () => result), + ...overrides, + }; + const registration = registerMediaIpc(options); + const sent: [string, unknown][] = []; + const event: MediaIpcEvent = { + sender: { send: (channel, payload) => sent.push([channel, payload]), isDestroyed: () => false }, + }; + const call = (channel: string, ...args: unknown[]) => handlers.get(channel)!(event, ...args); + return { handlers, helper, options, registration, call, sent }; +} + +describe("media IPC on platforms without Apple Photos", () => { + it.each(["linux", "win32"])( + "reports %s as unavailable and rejects sync and cancel", + async (platform) => { + const { call, options, handlers } = setup({ platform }); + expect([...handlers.keys()].sort()).toEqual( + [ + MEDIA_PHOTOS_CHANNELS.cancel, + MEDIA_PHOTOS_CHANNELS.status, + MEDIA_PHOTOS_CHANNELS.sync, + ].sort(), + ); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toEqual({ available: false }); + expect(() => call(MEDIA_PHOTOS_CHANNELS.sync, { spaceId: null })).toThrow(/macOS/); + expect(() => call(MEDIA_PHOTOS_CHANNELS.cancel)).toThrow(/macOS/); + expect(options.runSync).not.toHaveBeenCalled(); + expect(options.helper!.authorize).not.toHaveBeenCalled(); + }, + ); + + it("treats a macOS build without the helper as unavailable", async () => { + const { call } = setup({ helper: null }); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toEqual({ available: false }); + expect(() => call(MEDIA_PHOTOS_CHANNELS.sync, {})).toThrow(/macOS/); + }); + + it("treats a helper that cannot start as unavailable", async () => { + const { call, helper } = setup(); + vi.mocked(helper.authorize).mockRejectedValueOnce(new PhotosHelperError("unavailable", "x")); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toEqual({ available: false }); + }); +}); + +describe("media IPC on macOS", () => { + it("reads status without prompting", async () => { + const { call, helper } = setup(); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toEqual({ + available: true, + authorization: "authorized", + syncing: false, + }); + expect(helper.authorize).toHaveBeenCalledWith(false); + }); + + it("runs one sync at a time, forwards progress and resolves with the outcome", async () => { + let finish: (value: PhotosSyncResult) => void = () => {}; + const runSync = vi.fn( + ({ onProgress }: Parameters[0]) => + new Promise((resolve) => { + onProgress({ phase: "indexing", indexed: 1, uploadsDone: 0, uploadsTotal: 0 }); + finish = resolve; + }), + ); + const { call, sent } = setup({ runSync }); + const first = call(MEDIA_PHOTOS_CHANNELS.sync, { spaceId: "space_1" }); + const second = call(MEDIA_PHOTOS_CHANNELS.sync, { spaceId: "space_1" }); + expect(second).toBe(first); + expect(runSync).toHaveBeenCalledTimes(1); + expect(runSync.mock.calls[0]?.[0].spaceId).toBe("space_1"); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toMatchObject({ syncing: true }); + expect(sent).toEqual([ + [ + MEDIA_PHOTOS_CHANNELS.progress, + { phase: "indexing", indexed: 1, uploadsDone: 0, uploadsTotal: 0 }, + ], + ]); + finish(result); + await expect(first).resolves.toEqual({ ok: true, result }); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toMatchObject({ syncing: false }); + }); + + it("cancels the running sync", async () => { + const runSync = vi.fn( + ({ signal }: Parameters[0]) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => + reject(new PhotosSyncError("cancelled", "Sync cancelled.")), + ); + }), + ); + const { call } = setup({ runSync }); + const running = call(MEDIA_PHOTOS_CHANNELS.sync, {}); + await call(MEDIA_PHOTOS_CHANNELS.cancel); + await expect(running).resolves.toEqual({ + ok: false, + code: "cancelled", + message: "Sync cancelled.", + }); + }); + + it("rejects untrusted senders and malformed requests", async () => { + const { call } = setup({ isTrusted: () => false }); + expect(() => call(MEDIA_PHOTOS_CHANNELS.sync, {})).toThrow("Not allowed."); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).rejects.toThrow("Not allowed."); + expect(() => parseSyncRequest(null)).toThrow(); + expect(() => parseSyncRequest({ spaceId: "../x" })).toThrow(); + expect(() => parseSyncRequest({ spaceId: 4 })).toThrow(); + expect(parseSyncRequest({})).toEqual({ spaceId: null }); + }); + + it("never forwards unexpected error text to the renderer", () => { + expect(outcomeForError(new Error("/Users/example/secret path"))).toEqual({ + ok: false, + code: "failed", + message: "Apple Photos sync failed.", + }); + expect( + outcomeForError( + new PhotosHelperError("not_authorized", "Photos access has not been granted."), + ), + ).toMatchObject({ + code: "not_authorized", + }); + }); +}); diff --git a/apps/desktop/src/media-ipc.ts b/apps/desktop/src/media-ipc.ts new file mode 100644 index 0000000..bb8e109 --- /dev/null +++ b/apps/desktop/src/media-ipc.ts @@ -0,0 +1,135 @@ +import type { PhotosAuthorization, PhotosHelper } from "./media-photos-helper.js"; +import { PhotosHelperError } from "./media-photos-helper.js"; +import type { PhotosSyncProgress, PhotosSyncResult } from "./media-photos-sync.js"; +import { PhotosSyncError } from "./media-photos-sync.js"; + +/* + * IPC for the Apple Photos source. The renderer only expresses intent (sync, cancel) + * and renders state; the main process owns the helper, the session and the upload. + */ + +export const MEDIA_PHOTOS_CHANNELS = { + status: "desktop.media.photos.status", + sync: "desktop.media.photos.sync", + cancel: "desktop.media.photos.cancel", + progress: "desktop.media.photos.progress", +} as const; + +export type PhotosSourceStatus = + | { available: false } + | { available: true; authorization: PhotosAuthorization; syncing: boolean }; + +export type PhotosSyncOutcome = + | { ok: true; result: PhotosSyncResult } + | { ok: false; code: string; message: string }; + +/** Renderer-facing bridge, exposed as `rakazoDesktop.media.photos`. */ +export interface RakazoDesktopMediaPhotos { + status: () => Promise; + /** Asks for Photos access on first use, then indexes and uploads what the server needs. */ + sync: (request: { spaceId: string | null }) => Promise; + cancel: () => Promise; + onProgress: (listener: (progress: PhotosSyncProgress) => void) => () => void; +} + +export interface MediaIpcEvent { + sender: { send(channel: string, payload: unknown): void; isDestroyed(): boolean }; +} + +export interface MediaIpcMain { + handle(channel: string, listener: (event: MediaIpcEvent, ...args: unknown[]) => unknown): void; +} + +export interface MediaIpcOptions { + ipcMain: MediaIpcMain; + platform: string; + /** Null when the helper binary is missing (unbuilt dev checkout). */ + helper: PhotosHelper | null; + /** Same gate as the other app-window handlers: main window, main frame. */ + isTrusted: (event: MediaIpcEvent) => boolean; + runSync: (request: { + spaceId: string | null; + signal: AbortSignal; + onProgress: (progress: PhotosSyncProgress) => void; + }) => Promise; +} + +const UNAVAILABLE = "Apple Photos is only available in the macOS app."; +const SPACE_ID = /^[A-Za-z0-9_-]{1,200}$/; + +export function parseSyncRequest(value: unknown): { spaceId: string | null } { + if (typeof value !== "object" || value === null) throw new Error("Invalid sync request."); + const spaceId = (value as { spaceId?: unknown }).spaceId ?? null; + if (spaceId !== null && (typeof spaceId !== "string" || !SPACE_ID.test(spaceId))) { + throw new Error("Invalid sync request."); + } + return { spaceId }; +} + +/** Registers the handlers; `cancel()` stops a running sync, e.g. before quit. */ +export function registerMediaIpc(options: MediaIpcOptions): { cancel: () => void } { + const available = options.platform === "darwin" && options.helper !== null; + let running: { controller: AbortController; done: Promise } | null = null; + + const requireTrusted = (event: MediaIpcEvent) => { + if (!options.isTrusted(event)) throw new Error("Not allowed."); + }; + + options.ipcMain.handle(MEDIA_PHOTOS_CHANNELS.status, async (event) => { + requireTrusted(event); + if (!available) return { available: false } satisfies PhotosSourceStatus; + try { + const authorization = await options.helper!.authorize(false); + return { available: true, authorization, syncing: running !== null }; + } catch (error) { + if (error instanceof PhotosHelperError && error.code === "unavailable") { + return { available: false } satisfies PhotosSourceStatus; + } + throw error; + } + }); + + options.ipcMain.handle(MEDIA_PHOTOS_CHANNELS.sync, (event, request: unknown) => { + requireTrusted(event); + if (!available) throw new Error(UNAVAILABLE); + const { spaceId } = parseSyncRequest(request); + // A second click joins the running sync instead of starting a parallel one. + if (running) return running.done; + const controller = new AbortController(); + const done = options + .runSync({ + spaceId, + signal: controller.signal, + onProgress: (progress) => { + if (!event.sender.isDestroyed()) { + event.sender.send(MEDIA_PHOTOS_CHANNELS.progress, progress); + } + }, + }) + .then( + (result): PhotosSyncOutcome => ({ ok: true, result }), + (error: unknown): PhotosSyncOutcome => outcomeForError(error), + ) + .finally(() => { + if (running?.controller === controller) running = null; + }); + running = { controller, done }; + return done; + }); + + options.ipcMain.handle(MEDIA_PHOTOS_CHANNELS.cancel, (event) => { + requireTrusted(event); + if (!available) throw new Error(UNAVAILABLE); + running?.controller.abort(); + }); + + return { cancel: () => running?.controller.abort() }; +} + +/** Only our own fixed messages reach the renderer; anything else is generic. */ +export function outcomeForError(error: unknown): PhotosSyncOutcome { + if (error instanceof PhotosSyncError || error instanceof PhotosHelperError) { + return { ok: false, code: error.code, message: error.message }; + } + return { ok: false, code: "failed", message: "Apple Photos sync failed." }; +} diff --git a/apps/desktop/src/media-photos-helper.test.ts b/apps/desktop/src/media-photos-helper.test.ts new file mode 100644 index 0000000..513538a --- /dev/null +++ b/apps/desktop/src/media-photos-helper.test.ts @@ -0,0 +1,437 @@ +import { EventEmitter } from "node:events"; +import { mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { PassThrough } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { HelperProcess, SpawnHelper } from "./media-photos-helper.js"; +import { + changesArgs, + createPhotosHelper, + exportArgs, + helperEnv, + listArgs, + PhotosHelperError, + parseAssetOutput, + parseAuthorizeOutput, + parseChangesOutput, + parseExportOutput, + parseHelperError, + parseListOutput, + parseProgressLine, + photosHelperCandidates, + resolvePhotosHelperBinary, + runPhotosHelper, +} from "./media-photos-helper.js"; + +const item = (overrides: Record = {}) => ({ + externalId: "9F2A61B0-0000-4000-8000-000000000001/L0/001", + kind: "video", + capturedAt: "2024-01-02T03:04:05.500Z", + durationMs: 12_346, + width: 1920, + height: 1080, + fps: null, + gpsLat: 52.52, + gpsLon: 13.4, + deviceModel: null, + albumNames: ["Trip"], + isFavorite: true, + fileName: "IMG_0001.MOV", + ...overrides, +}); + +const listJson = (items: unknown[], nextCursor: unknown = null, changeToken: unknown = "tok") => + JSON.stringify({ items, nextCursor, changeToken }); + +function expectCode(run: () => unknown, code: string) { + try { + run(); + } catch (error) { + expect(error).toBeInstanceOf(PhotosHelperError); + expect((error as PhotosHelperError).code).toBe(code); + return; + } + throw new Error("expected a PhotosHelperError"); +} + +describe("photos helper output parsing", () => { + it("accepts a list page that matches the shared item contract", () => { + const page = parseListOutput( + listJson([item(), item({ kind: "photo", durationMs: null })], "c1"), + 2, + ); + expect(page.nextCursor).toBe("c1"); + expect(page.changeToken).toBe("tok"); + expect(page.items).toHaveLength(2); + expect(page.items[0]).toEqual(item()); + }); + + it("accepts the last page of an empty library", () => { + expect(parseListOutput(listJson([], null, null), 500)).toEqual({ + items: [], + nextCursor: null, + changeToken: null, + }); + }); + + it.each([ + ["not JSON", "items: []"], + ["an array document", "[]"], + ["truncated JSON", '{"items":[{"externalId":'], + ["missing items", JSON.stringify({ nextCursor: null, changeToken: null })], + ["an item missing a key", listJson([{ ...item(), width: undefined }])], + ["an unknown kind", listJson([item({ kind: "audio" })])], + ["out-of-range GPS", listJson([item({ gpsLat: 91 })])], + ["a non-UTC-parsable date", listJson([item({ capturedAt: "yesterday" })])], + ["a negative duration", listJson([item({ durationMs: -1 })])], + ["an empty external id", listJson([item({ externalId: "" })])], + ["an empty cursor", listJson([item()], "")], + ["a numeric cursor", listJson([item()], 3)], + ["too many items", listJson([item(), item(), item()])], + ])("rejects %s", (_name, text) => { + expectCode(() => parseListOutput(text, 2), "invalid_output"); + }); + + it("parses authorization states and rejects unknown ones", () => { + expect(parseAuthorizeOutput('{"status":"limited"}')).toBe("limited"); + expect(parseAuthorizeOutput('{"status":"notDetermined"}')).toBe("notDetermined"); + expectCode(() => parseAuthorizeOutput('{"status":"maybe"}'), "invalid_output"); + expectCode(() => parseAuthorizeOutput(""), "invalid_output"); + }); + + it("parses asset details", () => { + const detail = parseAssetOutput( + JSON.stringify({ + item: item(), + modifiedAt: "2024-02-01T00:00:00.000Z", + hidden: false, + resources: [{ type: "video", fileName: "IMG_0001.MOV", uti: "com.apple.quicktime-movie" }], + }), + ); + expect(detail.resources[0]?.type).toBe("video"); + expectCode( + () => parseAssetOutput(JSON.stringify({ item: item(), modifiedAt: null, resources: [] })), + "invalid_output", + ); + }); + + it("parses changes and requires a token unless a full relisting is needed", () => { + expect( + parseChangesOutput( + JSON.stringify({ + token: "t2", + changed: [item()], + deleted: ["x/L0/001"], + fullResyncRequired: false, + }), + ), + ).toMatchObject({ token: "t2", deleted: ["x/L0/001"], fullResyncRequired: false }); + expect( + parseChangesOutput( + JSON.stringify({ token: null, changed: [], deleted: [], fullResyncRequired: true }), + ).fullResyncRequired, + ).toBe(true); + expectCode( + () => + parseChangesOutput( + JSON.stringify({ token: null, changed: [], deleted: [], fullResyncRequired: false }), + ), + "invalid_output", + ); + expectCode( + () => + parseChangesOutput( + JSON.stringify({ token: "t", changed: [], deleted: [7], fullResyncRequired: false }), + ), + "invalid_output", + ); + }); + + it("maps helper error lines to codes", () => { + const stderr = + '{"progress":0.5}\n{"error":{"code":"not_authorized","message":"Photos access has not been granted."}}\n'; + const error = parseHelperError(stderr, 77); + expect(error.code).toBe("not_authorized"); + expect(error.message).toBe("Photos access has not been granted."); + expect(parseHelperError('{"error":{"code":"internal","message":"x"}}', 70).code).toBe("failed"); + expect(parseHelperError("Segmentation fault", 139).message).toContain("139"); + expect(parseHelperError('{"error":"oops"}', 1).code).toBe("failed"); + }); + + it("reads progress lines only", () => { + expect(parseProgressLine('{"progress":0.42}')).toBe(0.42); + expect(parseProgressLine('{"progress":2}')).toBeNull(); + expect(parseProgressLine('{"error":{"code":"usage"}}')).toBeNull(); + expect(parseProgressLine('{"progress":')).toBeNull(); + }); +}); + +describe("photos helper export result", () => { + let dir: string; + let other: string; + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "rakazo-photos-export-")); + other = await mkdtemp(path.join(tmpdir(), "rakazo-photos-other-")); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + await rm(other, { recursive: true, force: true }); + }); + + const result = (file: string, fileName = path.basename(file)) => + JSON.stringify({ path: file, fileName, sizeBytes: 3, sha256: "a".repeat(64) }); + + it("accepts a file directly inside the export folder", async () => { + const file = path.join(dir, "IMG_0001.MOV"); + await writeFile(file, "abc"); + await expect(parseExportOutput(result(file), dir)).resolves.toMatchObject({ + fileName: "IMG_0001.MOV", + sizeBytes: 3, + }); + }); + + it("rejects files outside the export folder, including through symlinks", async () => { + const outside = path.join(other, "secret.txt"); + await writeFile(outside, "abc"); + await expect(parseExportOutput(result(outside), dir)).rejects.toMatchObject({ + code: "path_rejected", + }); + const link = path.join(dir, "link.MOV"); + await symlink(outside, link); + await expect(parseExportOutput(result(link), dir)).rejects.toMatchObject({ + code: "path_rejected", + }); + const nested = path.join(dir, "..", path.basename(other), "secret.txt"); + await expect(parseExportOutput(result(nested), dir)).rejects.toMatchObject({ + code: "path_rejected", + }); + }); + + it("rejects malformed results", async () => { + const file = path.join(dir, "a.jpg"); + await writeFile(file, "abc"); + for (const text of [ + JSON.stringify({ path: file, fileName: "a.jpg", sizeBytes: 3, sha256: "XYZ" }), + JSON.stringify({ path: "a.jpg", fileName: "a.jpg", sizeBytes: 3, sha256: "a".repeat(64) }), + JSON.stringify({ path: file, fileName: "a.jpg", sizeBytes: -1, sha256: "a".repeat(64) }), + JSON.stringify({ + path: path.join(dir, "missing.jpg"), + fileName: "missing.jpg", + sizeBytes: 3, + sha256: "a".repeat(64), + }), + ]) { + await expect(parseExportOutput(text, dir)).rejects.toMatchObject({ code: "invalid_output" }); + } + await expect(parseExportOutput(result(file, "b.jpg"), dir)).rejects.toMatchObject({ + code: "path_rejected", + }); + }); +}); + +describe("photos helper argv", () => { + it("builds fixed argv arrays", () => { + expect(listArgs({ cursor: null, limit: 500 })).toEqual(["list", "--limit", "500"]); + expect(listArgs({ cursor: "abc_-", limit: 10, since: "2024-01-01T00:00:00Z" })).toEqual([ + "list", + "--limit", + "10", + "--cursor", + "abc_-", + "--since", + "2024-01-01T00:00:00Z", + ]); + expect(exportArgs("ID/L0/001", "/tmp/rakazo-photos-1")).toEqual([ + "export", + "ID/L0/001", + "--dest", + "/tmp/rakazo-photos-1", + "--original", + ]); + expect(changesArgs("tok")).toEqual(["changes", "--token", "tok"]); + }); + + it("keeps shell syntax as inert data and rejects option-like values", () => { + // No shell runs, so metacharacters stay one literal argument. + expect(exportArgs("$(rm -rf ~);`x`", "/tmp/a b")[1]).toBe("$(rm -rf ~);`x`"); + expectCode(() => exportArgs("--dest", "/tmp"), "usage"); + expectCode(() => listArgs({ cursor: "--limit", limit: 1 }), "usage"); + expectCode(() => changesArgs("a\nb"), "usage"); + expectCode(() => listArgs({ cursor: null, limit: 501 }), "usage"); + expectCode(() => exportArgs("ID", "relative/dir"), "path_rejected"); + expectCode(() => exportArgs("ID", "/tmp/../etc"), "path_rejected"); + }); + + it("passes only the environment PhotoKit needs", () => { + expect( + helperEnv({ HOME: "/Users/example", TMPDIR: "/tmp", OPENAI_API_KEY: "secret", PATH: "/bin" }), + ).toEqual({ HOME: "/Users/example", TMPDIR: "/tmp" }); + }); +}); + +describe("photos helper binary", () => { + it("is only available on macOS", () => { + const options = { + packaged: true, + resourcesPath: "/App/Resources", + appPath: "/app", + exists: () => true, + }; + expect(resolvePhotosHelperBinary({ ...options, platform: "linux" })).toBeNull(); + expect(resolvePhotosHelperBinary({ ...options, platform: "win32" })).toBeNull(); + expect(resolvePhotosHelperBinary({ ...options, platform: "darwin" })).toBe( + "/App/Resources/bin/photos-helper", + ); + }); + + it("falls back to the development build and reports a missing build", () => { + const options = { + platform: "darwin", + packaged: false, + resourcesPath: "/r", + appPath: "/repo/apps/desktop", + }; + expect(photosHelperCandidates(options)).toEqual([ + "/repo/apps/desktop/native/photos-helper/.build/apple/Products/Release/photos-helper", + "/repo/apps/desktop/native/photos-helper/.build/release/photos-helper", + ]); + expect( + resolvePhotosHelperBinary({ + ...options, + exists: (file) => file.endsWith(".build/release/photos-helper"), + }), + ).toBe("/repo/apps/desktop/native/photos-helper/.build/release/photos-helper"); + expect(resolvePhotosHelperBinary({ ...options, exists: () => false })).toBeNull(); + }); +}); + +class FakeProcess extends EventEmitter implements HelperProcess { + stdout = new PassThrough(); + stderr = new PassThrough(); + kill = vi.fn(() => { + queueMicrotask(() => this.emit("close", null)); + return true; + }); + exit(code: number, stdout = "", stderr = "") { + if (stdout) this.stdout.write(stdout); + if (stderr) this.stderr.write(stderr); + setImmediate(() => this.emit("close", code)); + } +} + +function fakeSpawn() { + const processes: FakeProcess[] = []; + const spawn = vi.fn(() => { + const child = new FakeProcess(); + processes.push(child); + return child; + }); + return { spawn, processes }; +} + +describe("photos helper process", () => { + it("spawns the binary with argv and no shell", async () => { + const { spawn, processes } = fakeSpawn(); + const helper = createPhotosHelper("/App/Resources/bin/photos-helper", spawn); + const pending = helper.list({ cursor: null, limit: 2 }); + processes[0]!.exit(0, listJson([item()])); + await expect(pending).resolves.toMatchObject({ items: [item()] }); + expect(spawn).toHaveBeenCalledWith( + "/App/Resources/bin/photos-helper", + ["list", "--limit", "2"], + expect.objectContaining({ shell: false, stdio: ["ignore", "pipe", "pipe"] }), + ); + }); + + it("maps a failing exit to the helper's error code", async () => { + const { spawn, processes } = fakeSpawn(); + const pending = createPhotosHelper("/bin/photos-helper", spawn).asset("ID/L0/001"); + processes[0]!.exit( + 66, + "", + '{"error":{"code":"not_found","message":"No photo or video with that identifier."}}\n', + ); + await expect(pending).rejects.toMatchObject({ code: "not_found" }); + }); + + it("reports export progress and validates the result", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "rakazo-photos-run-")); + try { + const file = path.join(dir, "IMG_0001.MOV"); + await writeFile(file, "abc"); + const { spawn, processes } = fakeSpawn(); + const progress: number[] = []; + const pending = createPhotosHelper("/bin/photos-helper", spawn).exportOriginal("ID", dir, { + onProgress: (value) => progress.push(value), + }); + const child = processes[0]!; + child.stderr.write('{"progress":0}\n{"progress":0.5'); + child.stderr.write('}\nnoise\n{"progress":1}\n'); + child.exit( + 0, + JSON.stringify({ + path: file, + fileName: "IMG_0001.MOV", + sizeBytes: 3, + sha256: "b".repeat(64), + }), + ); + await expect(pending).resolves.toMatchObject({ sizeBytes: 3 }); + expect(progress).toEqual([0, 0.5, 1]); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it("kills the helper on abort and on timeout", async () => { + const { spawn, processes } = fakeSpawn(); + const controller = new AbortController(); + const aborted = runPhotosHelper("/bin/h", ["list"], { + spawn, + timeoutMs: 60_000, + signal: controller.signal, + }); + controller.abort(); + await expect(aborted).rejects.toMatchObject({ code: "cancelled" }); + expect(processes[0]!.kill).toHaveBeenCalledWith("SIGTERM"); + + const timedOut = runPhotosHelper("/bin/h", ["list"], { spawn, timeoutMs: 5 }); + await expect(timedOut).rejects.toMatchObject({ code: "timeout" }); + expect(processes[1]!.kill).toHaveBeenCalled(); + }); + + it("fails a silent export through the idle limit", async () => { + const { spawn, processes } = fakeSpawn(); + const pending = runPhotosHelper("/bin/h", ["export"], { + spawn, + timeoutMs: 60_000, + idleTimeoutMs: 5, + }); + await expect(pending).rejects.toMatchObject({ code: "timeout" }); + expect(processes[0]!.kill).toHaveBeenCalled(); + }); + + it("does not start when already aborted and reports spawn failures", async () => { + const { spawn, processes } = fakeSpawn(); + const controller = new AbortController(); + controller.abort(); + await expect( + runPhotosHelper("/bin/h", ["list"], { spawn, timeoutMs: 1000, signal: controller.signal }), + ).rejects.toMatchObject({ code: "cancelled" }); + expect(spawn).not.toHaveBeenCalled(); + + const pending = runPhotosHelper("/bin/h", ["list"], { spawn, timeoutMs: 1000 }); + processes[0]!.emit("error", new Error("ENOENT")); + await expect(pending).rejects.toMatchObject({ code: "unavailable" }); + }); + + it("rejects oversized output", async () => { + const { spawn, processes } = fakeSpawn(); + const pending = runPhotosHelper("/bin/h", ["list"], { spawn, timeoutMs: 1000 }); + processes[0]!.stdout.write(Buffer.alloc(33 * 1024 * 1024)); + await expect(pending).rejects.toMatchObject({ code: "invalid_output" }); + }); +}); diff --git a/apps/desktop/src/media-photos-helper.ts b/apps/desktop/src/media-photos-helper.ts new file mode 100644 index 0000000..f10e402 --- /dev/null +++ b/apps/desktop/src/media-photos-helper.ts @@ -0,0 +1,517 @@ +import { spawn as nodeSpawn } from "node:child_process"; +import { realpath } from "node:fs/promises"; +import path from "node:path"; +import type { Readable } from "node:stream"; +import type { ClientMediaItem } from "@rakazo/contracts"; +import { ClientMediaItemSchema } from "@rakazo/contracts"; + +/* + * Client for the macOS `photos-helper` CLI (apps/desktop/native/photos-helper). The + * helper runs with a fixed argv and no shell; every byte it prints is validated before + * the app uses it, so a malformed or unexpected answer fails instead of being guessed at. + */ + +export type PhotosAuthorization = + | "authorized" + | "limited" + | "denied" + | "restricted" + | "notDetermined"; + +export interface PhotosListPage { + items: ClientMediaItem[]; + nextCursor: string | null; + /** Library change token from before this page was read. */ + changeToken: string | null; +} + +export interface PhotosAssetDetail { + item: ClientMediaItem; + modifiedAt: string | null; + hidden: boolean; + resources: { type: string; fileName: string; uti: string }[]; +} + +export interface PhotosChanges { + token: string | null; + changed: ClientMediaItem[]; + deleted: string[]; + fullResyncRequired: boolean; +} + +export interface PhotosExport { + path: string; + fileName: string; + sizeBytes: number; + sha256: string; +} + +export type PhotosHelperErrorCode = + | "unavailable" + | "usage" + | "not_authorized" + | "not_found" + | "path_rejected" + | "destination_exists" + | "network" + | "export_failed" + | "invalid_output" + | "timeout" + | "cancelled" + | "failed"; + +const HELPER_CODES = new Set([ + "usage", + "not_authorized", + "not_found", + "path_rejected", + "destination_exists", + "network", + "export_failed", +]); + +export class PhotosHelperError extends Error { + constructor( + readonly code: PhotosHelperErrorCode, + message: string, + ) { + super(message); + this.name = "PhotosHelperError"; + } +} + +export const PHOTOS_HELPER_NAME = "photos-helper"; +/** Matches the contract's batch limit for `media.sources.pushItems`. */ +export const PHOTOS_MAX_PAGE = 500; + +/** + * Packaged apps ship the universal helper in Resources/bin; development uses the + * SwiftPM output (universal `--arch` build first, then the host-arch build). + */ +export function photosHelperCandidates(options: { + packaged: boolean; + resourcesPath: string; + appPath: string; +}): string[] { + if (options.packaged) return [path.join(options.resourcesPath, "bin", PHOTOS_HELPER_NAME)]; + const build = path.join(options.appPath, "native", "photos-helper", ".build"); + return [ + path.join(build, "apple", "Products", "Release", PHOTOS_HELPER_NAME), + path.join(build, "release", PHOTOS_HELPER_NAME), + ]; +} + +/** Null off macOS or when the helper was not built; the Apple Photos source is then hidden. */ +export function resolvePhotosHelperBinary(options: { + platform: string; + packaged: boolean; + resourcesPath: string; + appPath: string; + exists: (file: string) => boolean; +}): string | null { + if (options.platform !== "darwin") return null; + return photosHelperCandidates(options).find((candidate) => options.exists(candidate)) ?? null; +} + +// --------------------------------------------------------------------------- +// argv +// --------------------------------------------------------------------------- + +export function authorizeArgs(prompt: boolean): string[] { + return prompt ? ["authorize"] : ["authorize", "--no-prompt"]; +} + +export function listArgs(request: { + cursor: string | null; + limit: number; + since?: string | null; +}): string[] { + if (!Number.isInteger(request.limit) || request.limit < 1 || request.limit > PHOTOS_MAX_PAGE) { + throw new PhotosHelperError("usage", "List limit is out of range."); + } + const args = ["list", "--limit", String(request.limit)]; + if (request.cursor !== null) args.push("--cursor", opaqueArgument(request.cursor)); + if (request.since) args.push("--since", opaqueArgument(request.since)); + return args; +} + +export function assetArgs(localIdentifier: string): string[] { + return ["asset", identifierArgument(localIdentifier)]; +} + +export function exportArgs(localIdentifier: string, destination: string): string[] { + if (!path.isAbsolute(destination) || destination.split(/[\\/]/).includes("..")) { + throw new PhotosHelperError("path_rejected", "Export directory must be absolute."); + } + return ["export", identifierArgument(localIdentifier), "--dest", destination, "--original"]; +} + +export function changesArgs(token: string): string[] { + return ["changes", "--token", opaqueArgument(token)]; +} + +/** Identifiers come from the helper itself; a leading dash would read as an option. */ +function identifierArgument(value: string): string { + if (value === "" || value.length > 200 || value.startsWith("-") || hasControl(value)) { + throw new PhotosHelperError("usage", "Invalid Photos identifier."); + } + return value; +} + +function opaqueArgument(value: string): string { + if (value === "" || value.length > 4096 || value.startsWith("-") || hasControl(value)) { + throw new PhotosHelperError("usage", "Invalid helper argument."); + } + return value; +} + +function hasControl(value: string): boolean { + for (const char of value) { + const code = char.codePointAt(0) ?? 0; + if (code < 0x20 || code === 0x7f) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// Output parsing +// --------------------------------------------------------------------------- + +function invalid(what: string): never { + throw new PhotosHelperError("invalid_output", `Photos helper returned invalid ${what}.`); +} + +function parseDocument(text: string): Record { + let value: unknown; + try { + value = JSON.parse(text); + } catch { + invalid("JSON"); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) invalid("JSON"); + return value as Record; +} + +function nullableString(value: unknown, what: string, max = 4096): string | null { + if (value === null) return null; + if (typeof value !== "string" || value === "" || value.length > max) invalid(what); + return value; +} + +/** Each item must satisfy the shared contract schema exactly as the server will parse it. */ +function parseItems(value: unknown, what: string): ClientMediaItem[] { + if (!Array.isArray(value) || value.length > 10_000) invalid(what); + return value.map((entry) => { + if (typeof entry !== "object" || entry === null) invalid(what); + const parsed = ClientMediaItemSchema.safeParse(entry); + if (!parsed.success) invalid(what); + return parsed.data; + }); +} + +const AUTHORIZATION = new Set([ + "authorized", + "limited", + "denied", + "restricted", + "notDetermined", +]); + +export function parseAuthorizeOutput(text: string): PhotosAuthorization { + const status = parseDocument(text).status; + if (typeof status !== "string" || !AUTHORIZATION.has(status as PhotosAuthorization)) { + invalid("authorization status"); + } + return status as PhotosAuthorization; +} + +export function parseListOutput(text: string, limit: number): PhotosListPage { + const document = parseDocument(text); + const items = parseItems(document.items, "list items"); + if (items.length > limit) invalid("list page size"); + const nextCursor = nullableString(document.nextCursor, "list cursor"); + const changeToken = nullableString(document.changeToken, "change token"); + return { items, nextCursor, changeToken }; +} + +export function parseAssetOutput(text: string): PhotosAssetDetail { + const document = parseDocument(text); + const [item] = parseItems([document.item], "asset"); + const modifiedAt = nullableString(document.modifiedAt, "asset date", 64); + if (modifiedAt !== null && Number.isNaN(Date.parse(modifiedAt))) invalid("asset date"); + if (typeof document.hidden !== "boolean") invalid("asset"); + if (!Array.isArray(document.resources) || document.resources.length > 50) invalid("resources"); + const resources = document.resources.map((resource) => { + if (typeof resource !== "object" || resource === null) invalid("resources"); + const { type, fileName, uti } = resource as Record; + if (typeof type !== "string" || typeof fileName !== "string" || typeof uti !== "string") { + invalid("resources"); + } + return { type, fileName, uti }; + }); + return { item: item!, modifiedAt, hidden: document.hidden, resources }; +} + +export function parseChangesOutput(text: string): PhotosChanges { + const document = parseDocument(text); + const token = nullableString(document.token, "change token"); + if (typeof document.fullResyncRequired !== "boolean") invalid("changes"); + const changed = parseItems(document.changed, "changed items"); + if (!Array.isArray(document.deleted) || document.deleted.length > 10_000) invalid("changes"); + const deleted = document.deleted.map((id) => { + if (typeof id !== "string" || id === "" || id.length > 200) invalid("deleted ids"); + return id; + }); + if (!document.fullResyncRequired && token === null) invalid("change token"); + return { token, changed, deleted, fullResyncRequired: document.fullResyncRequired }; +} + +/** + * Besides the shape, the file must be a direct child of the directory the app chose, + * after resolving symlinks, so a confused helper cannot point the upload elsewhere. + */ +export async function parseExportOutput(text: string, destination: string): Promise { + const document = parseDocument(text); + const { path: file, fileName, sizeBytes, sha256 } = document; + if ( + typeof file !== "string" || + !path.isAbsolute(file) || + typeof fileName !== "string" || + fileName === "" || + typeof sizeBytes !== "number" || + !Number.isSafeInteger(sizeBytes) || + sizeBytes < 0 || + typeof sha256 !== "string" || + !/^[0-9a-f]{64}$/.test(sha256) + ) { + invalid("export result"); + } + let real: string; + let realDestination: string; + try { + [real, realDestination] = await Promise.all([realpath(file), realpath(destination)]); + } catch { + invalid("export path"); + } + if (path.dirname(real) !== realDestination || path.basename(real) !== fileName) { + throw new PhotosHelperError("path_rejected", "Photos helper wrote outside the export folder."); + } + return { path: real, fileName, sizeBytes, sha256 }; +} + +/** The last `{"error":{code,message}}` line on stderr; unknown codes become `failed`. */ +export function parseHelperError(stderr: string, exitCode: number | null): PhotosHelperError { + const lines = stderr.split("\n").reverse(); + for (const line of lines) { + if (!line.startsWith("{")) continue; + try { + const error = (JSON.parse(line) as { error?: unknown }).error as + | { code?: unknown; message?: unknown } + | undefined; + if (typeof error?.code !== "string") continue; + const code = HELPER_CODES.has(error.code as PhotosHelperErrorCode) + ? (error.code as PhotosHelperErrorCode) + : "failed"; + const message = + typeof error.message === "string" && error.message.length <= 500 + ? error.message + : "Photos helper failed."; + return new PhotosHelperError(code, message); + } catch {} + } + return new PhotosHelperError("failed", `Photos helper exited with status ${exitCode ?? "?"}.`); +} + +/** `{"progress":0.42}` lines on stderr; anything else is ignored. */ +export function parseProgressLine(line: string): number | null { + if (!line.startsWith('{"progress"')) return null; + try { + const value = (JSON.parse(line) as { progress?: unknown }).progress; + return typeof value === "number" && value >= 0 && value <= 1 ? value : null; + } catch { + return null; + } +} + +// --------------------------------------------------------------------------- +// Process +// --------------------------------------------------------------------------- + +/** The subset of ChildProcess the runner needs, so tests can supply a fake. */ +export interface HelperProcess { + stdout: Readable | null; + stderr: Readable | null; + kill(signal?: NodeJS.Signals): boolean; + on(event: "error", listener: (error: Error) => void): unknown; + on(event: "close", listener: (code: number | null) => void): unknown; +} + +export type SpawnHelper = ( + binary: string, + args: string[], + options: { shell: false; env: Record; stdio: ["ignore", "pipe", "pipe"] }, +) => HelperProcess; + +export interface RunOptions { + signal?: AbortSignal; + /** Hard limit for the whole run. */ + timeoutMs: number; + /** Fails when stderr progress stops for this long (exports only). */ + idleTimeoutMs?: number; + onProgress?: (fraction: number) => void; +} + +const MAX_STDOUT_BYTES = 32 * 1024 * 1024; +const MAX_STDERR_BYTES = 1024 * 1024; +const ENV_ALLOWLIST = ["HOME", "USER", "LOGNAME", "TMPDIR", "LANG", "LC_ALL"]; + +/** PhotoKit needs the user's home; nothing else from the app environment is passed on. */ +export function helperEnv(env: NodeJS.ProcessEnv): Record { + const result: Record = {}; + for (const name of ENV_ALLOWLIST) { + const value = env[name]; + if (value) result[name] = value; + } + return result; +} + +export const defaultSpawnHelper: SpawnHelper = (binary, args, options) => + nodeSpawn(binary, args, options); + +export function runPhotosHelper( + binary: string, + args: string[], + options: RunOptions & { spawn?: SpawnHelper; env?: NodeJS.ProcessEnv }, +): Promise { + return new Promise((resolve, reject) => { + if (options.signal?.aborted) { + reject(new PhotosHelperError("cancelled", "Cancelled.")); + return; + } + const child = (options.spawn ?? defaultSpawnHelper)(binary, args, { + shell: false, + env: helperEnv(options.env ?? process.env), + stdio: ["ignore", "pipe", "pipe"], + }); + const stdout: Buffer[] = []; + let stdoutBytes = 0; + let stderr = ""; + let pendingLine = ""; + let settled = false; + let idleTimer: NodeJS.Timeout | undefined; + + const finish = (error: PhotosHelperError | null, output = "") => { + if (settled) return; + settled = true; + clearTimeout(timer); + clearTimeout(idleTimer); + options.signal?.removeEventListener("abort", abort); + if (error) reject(error); + else resolve(output); + }; + const stop = (error: PhotosHelperError) => { + child.kill("SIGTERM"); + finish(error); + }; + const abort = () => stop(new PhotosHelperError("cancelled", "Cancelled.")); + const timer = setTimeout( + () => stop(new PhotosHelperError("timeout", "Photos helper timed out.")), + options.timeoutMs, + ); + timer.unref?.(); + const armIdle = () => { + if (options.idleTimeoutMs === undefined) return; + clearTimeout(idleTimer); + idleTimer = setTimeout( + () => stop(new PhotosHelperError("timeout", "Photos export stalled.")), + options.idleTimeoutMs, + ); + idleTimer.unref?.(); + }; + armIdle(); + options.signal?.addEventListener("abort", abort, { once: true }); + + child.stdout?.on("data", (chunk: Buffer) => { + stdoutBytes += chunk.byteLength; + if (stdoutBytes > MAX_STDOUT_BYTES) { + stop(new PhotosHelperError("invalid_output", "Photos helper output is too large.")); + return; + } + stdout.push(chunk); + }); + child.stderr?.on("data", (chunk: Buffer) => { + const text = chunk.toString("utf8"); + if (stderr.length < MAX_STDERR_BYTES) stderr += text; + const lines = (pendingLine + text).split("\n"); + pendingLine = lines.pop() ?? ""; + for (const line of lines) { + const progress = parseProgressLine(line.trim()); + if (progress !== null) { + armIdle(); + options.onProgress?.(progress); + } + } + }); + child.on("error", () => { + finish(new PhotosHelperError("unavailable", "Photos helper could not be started.")); + }); + child.on("close", (code) => { + if (code === 0) finish(null, Buffer.concat(stdout).toString("utf8")); + else finish(parseHelperError(stderr + pendingLine, code)); + }); + }); +} + +const QUERY_TIMEOUT_MS = 2 * 60_000; +/** The system permission dialog waits on the person. */ +const AUTHORIZE_TIMEOUT_MS = 10 * 60_000; +/** Large iCloud videos take long; a stalled download fails sooner through the idle limit. */ +const EXPORT_TIMEOUT_MS = 2 * 60 * 60_000; +const EXPORT_IDLE_TIMEOUT_MS = 5 * 60_000; + +export interface PhotosHelper { + authorize(prompt: boolean, signal?: AbortSignal): Promise; + list( + request: { cursor: string | null; limit: number; since?: string | null }, + signal?: AbortSignal, + ): Promise; + asset(localIdentifier: string, signal?: AbortSignal): Promise; + exportOriginal( + localIdentifier: string, + destination: string, + options?: { signal?: AbortSignal; onProgress?: (fraction: number) => void }, + ): Promise; + changes(token: string, signal?: AbortSignal): Promise; +} + +export function createPhotosHelper(binary: string, spawn?: SpawnHelper): PhotosHelper { + const run = (args: string[], options: RunOptions) => + runPhotosHelper(binary, args, { ...options, spawn }); + return { + async authorize(prompt, signal) { + const timeoutMs = prompt ? AUTHORIZE_TIMEOUT_MS : QUERY_TIMEOUT_MS; + return parseAuthorizeOutput(await run(authorizeArgs(prompt), { signal, timeoutMs })); + }, + async list(request, signal) { + const output = await run(listArgs(request), { signal, timeoutMs: QUERY_TIMEOUT_MS }); + return parseListOutput(output, request.limit); + }, + async asset(localIdentifier, signal) { + const output = await run(assetArgs(localIdentifier), { signal, timeoutMs: QUERY_TIMEOUT_MS }); + return parseAssetOutput(output); + }, + async exportOriginal(localIdentifier, destination, options = {}) { + const output = await run(exportArgs(localIdentifier, destination), { + signal: options.signal, + onProgress: options.onProgress, + timeoutMs: EXPORT_TIMEOUT_MS, + idleTimeoutMs: EXPORT_IDLE_TIMEOUT_MS, + }); + return parseExportOutput(output, destination); + }, + async changes(token, signal) { + const output = await run(changesArgs(token), { signal, timeoutMs: QUERY_TIMEOUT_MS }); + return parseChangesOutput(output); + }, + }; +} diff --git a/apps/desktop/src/media-photos-sync.test.ts b/apps/desktop/src/media-photos-sync.test.ts new file mode 100644 index 0000000..a1928a8 --- /dev/null +++ b/apps/desktop/src/media-photos-sync.test.ts @@ -0,0 +1,658 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { ClientMediaItem, MediaAsset, MediaUpload } from "@rakazo/contracts"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { PhotosHelper } from "./media-photos-helper.js"; +import type { MediaSyncApi, PhotosSyncState, PhotosSyncStore } from "./media-photos-sync.js"; +import { + createChunkUploader, + createPhotosSyncFileStore, + parsePhotosSyncState, + runPool, + syncApplePhotos, +} from "./media-photos-sync.js"; + +const sha = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); + +function libraryItem(index: number): ClientMediaItem { + return { + externalId: `ASSET-${String(index).padStart(5, "0")}/L0/001`, + kind: index % 3 === 0 ? "video" : "photo", + capturedAt: new Date(Date.UTC(2024, 0, 1) + index * 60_000).toISOString(), + durationMs: index % 3 === 0 ? 5000 : null, + width: 1920, + height: 1080, + fps: null, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + isFavorite: false, + fileName: `IMG_${index}.JPG`, + }; +} + +/** Pages a synthetic library the way the Swift helper does: opaque cursor = next index. */ +function fakeHelper(options: { size: number; contents?: (id: string) => string }) { + const library = Array.from({ length: options.size }, (_, index) => libraryItem(index)); + const exports: string[] = []; + let active = 0; + const stats = { maxActiveExports: 0 }; + const helper: PhotosHelper & { library: ClientMediaItem[] } = { + library, + authorize: vi.fn(async () => "authorized" as const), + list: vi.fn(async ({ cursor, limit }) => { + const start = cursor === null ? 0 : Number(cursor); + const items = library.slice(start, start + limit); + const next = start + limit < library.length ? String(start + limit) : null; + return { items, nextCursor: next, changeToken: `token-${library.length}` }; + }), + asset: vi.fn(), + changes: vi.fn(), + exportOriginal: vi.fn(async (id: string, destination: string, opts) => { + active += 1; + stats.maxActiveExports = Math.max(stats.maxActiveExports, active); + try { + await new Promise((resolve) => setTimeout(resolve, 5)); + opts?.signal?.throwIfAborted(); + const contents = options.contents?.(id) ?? `original bytes of ${id}`; + const file = path.join(destination, "IMG.MOV"); + await writeFile(file, contents); + exports.push(destination); + return { + path: file, + fileName: "IMG.MOV", + sizeBytes: Buffer.byteLength(contents), + sha256: sha(contents), + }; + } finally { + active -= 1; + } + }), + }; + return { helper, exports, stats }; +} + +function memoryStore(initial?: PhotosSyncState) { + let state: PhotosSyncState = initial ?? { version: 1, deviceId: "device-1", targets: {} }; + const writes: PhotosSyncState[] = []; + const store: PhotosSyncStore = { + read: async () => JSON.parse(JSON.stringify(state)) as PhotosSyncState, + write: async (next) => { + state = JSON.parse(JSON.stringify(next)) as PhotosSyncState; + writes.push(state); + }, + }; + return { store, writes, current: () => state }; +} + +function asset(id: string, externalId: string | undefined, ingestState: MediaAsset["ingestState"]) { + return { + id, + sourceId: "source-1", + kind: "video", + storage: "remote", + capturedAt: null, + durationMs: null, + width: null, + height: null, + fps: null, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + isFavorite: false, + ingestState, + hasProxy: false, + hasThumbnail: false, + qualityScore: null, + createdAt: "2024-01-01T00:00:00.000Z", + ...(externalId === undefined ? {} : { externalId }), + } as MediaAsset; +} + +function fakeApi( + options: { + assets?: MediaAsset[]; + chunkBytes?: number; + uploads?: Record; + } = {}, +) { + const uploads = new Map(Object.entries(options.uploads ?? {})); + const pushes: { items: number; syncToken: string | null; complete: boolean }[] = []; + const chunkBytes = options.chunkBytes ?? 4; + const api = { + media: { + sources: { + registerClient: vi.fn(async (input: { deviceId: string }) => ({ + id: "source-1", + kind: "apple_photos" as const, + label: "Apple Photos", + status: "idle" as const, + assetCount: 0, + lastSyncedAt: null, + error: null, + createdAt: "2024-01-01T00:00:00.000Z", + deviceId: input.deviceId, + })), + pushItems: vi.fn( + async (input: { items: unknown[]; syncToken: string | null; complete: boolean }) => { + expect(input.items.length).toBeGreaterThan(0); + expect(input.items.length).toBeLessThanOrEqual(500); + pushes.push({ + items: input.items.length, + syncToken: input.syncToken, + complete: input.complete, + }); + return { created: input.items.length, updated: 0, unchanged: 0 }; + }, + ), + }, + assets: { + search: vi.fn(async (input: { cursor?: string | null }) => { + const all = options.assets ?? []; + // Two pages to exercise pagination. + const half = Math.ceil(all.length / 2); + return input.cursor === "p2" + ? { items: all.slice(half), nextCursor: null } + : { items: all.slice(0, half), nextCursor: all.length > 1 ? "p2" : null }; + }), + }, + uploads: { + begin: vi.fn(async (input: { assetId: string; sizeBytes: number }) => { + const upload: MediaUpload = { + id: `upload-${input.assetId}`, + assetId: input.assetId, + sizeBytes: input.sizeBytes, + chunkBytes, + receivedChunks: [], + status: "open", + }; + uploads.set(upload.id, upload); + return upload; + }), + status: vi.fn(async (input: { uploadId: string }) => { + const upload = uploads.get(input.uploadId); + if (!upload) throw Object.assign(new Error("Not found"), { code: "NOT_FOUND" }); + return upload; + }), + complete: vi.fn(async (input: { uploadId: string }) => { + const upload = uploads.get(input.uploadId)!; + return { ...upload, status: "complete" as const }; + }), + }, + }, + }; + return { api: api as unknown as MediaSyncApi & typeof api, pushes }; +} + +let tempRoot: string; + +beforeEach(async () => { + tempRoot = await mkdtemp(path.join(tmpdir(), "rakazo-photos-sync-")); +}); + +afterEach(async () => { + await rm(tempRoot, { recursive: true, force: true }); +}); + +const baseOptions = () => ({ + target: "https://rakazo.example.com#space_1", + label: "Apple Photos", + tempRoot, + signal: new AbortController().signal, +}); + +describe("Apple Photos metadata sync", () => { + it("registers once, pushes batches of at most 500 and completes the listing", async () => { + const { helper } = fakeHelper({ size: 1203 }); + const { api, pushes } = fakeApi(); + const { store, current } = memoryStore(); + const chunks = vi.fn(); + + const result = await syncApplePhotos({ + ...baseOptions(), + helper, + api, + store, + uploadChunk: chunks, + }); + + expect(api.media.sources.registerClient).toHaveBeenCalledWith( + { kind: "apple_photos", label: "Apple Photos", deviceId: "device-1" }, + expect.anything(), + ); + expect(pushes).toEqual([ + { items: 500, syncToken: null, complete: false }, + { items: 500, syncToken: null, complete: false }, + { items: 203, syncToken: "token-1203", complete: true }, + ]); + expect(result).toMatchObject({ indexed: 1203, created: 1203, uploaded: 0, fullListing: true }); + expect(current().targets["https://rakazo.example.com#space_1"]).toEqual({ + sourceId: "source-1", + listing: null, + changeToken: "token-1203", + uploads: {}, + }); + + // The next run reads changes only and does not register again. + vi.mocked(helper.changes).mockResolvedValueOnce({ + token: "token-next", + changed: [libraryItem(1), libraryItem(2)], + deleted: [], + fullResyncRequired: false, + }); + const second = await syncApplePhotos({ + ...baseOptions(), + helper, + api, + store, + uploadChunk: chunks, + }); + expect(api.media.sources.registerClient).toHaveBeenCalledTimes(1); + expect(helper.changes).toHaveBeenCalledWith("token-1203", expect.anything()); + expect(pushes.at(-1)).toEqual({ items: 2, syncToken: "token-next", complete: false }); + expect(second.fullListing).toBe(false); + expect(current().targets["https://rakazo.example.com#space_1"]?.changeToken).toBe("token-next"); + }); + + it("persists the cursor after every batch and resumes an interrupted listing", async () => { + const { helper } = fakeHelper({ size: 1100 }); + const { api, pushes } = fakeApi(); + const { store, writes, current } = memoryStore(); + vi.mocked(api.media.sources.pushItems).mockImplementationOnce(async (input) => { + pushes.push({ + items: input.items.length, + syncToken: input.syncToken, + complete: input.complete, + }); + return { created: input.items.length, updated: 0, unchanged: 0 }; + }); + vi.mocked(api.media.sources.pushItems).mockRejectedValueOnce(new Error("network down")); + + await expect( + syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }), + ).rejects.toThrow("network down"); + const interrupted = current().targets["https://rakazo.example.com#space_1"]!; + expect(interrupted.listing).toMatchObject({ cursor: "500", token: "token-1100" }); + expect(interrupted.listing?.lastItem?.externalId).toBe(libraryItem(499).externalId); + expect(writes.length).toBeGreaterThan(1); + + await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }); + expect(vi.mocked(helper.list).mock.calls.at(-2)?.[0]).toMatchObject({ cursor: "500" }); + expect(pushes.slice(1)).toEqual([ + { items: 500, syncToken: null, complete: false }, + { items: 100, syncToken: "token-1100", complete: true }, + ]); + expect(current().targets["https://rakazo.example.com#space_1"]).toMatchObject({ + listing: null, + changeToken: "token-1100", + }); + }); + + it("closes a resumed listing whose last page came back empty", async () => { + const { helper } = fakeHelper({ size: 0 }); + const { api, pushes } = fakeApi(); + const last = libraryItem(7); + const { store } = memoryStore({ + version: 1, + deviceId: "device-1", + targets: { + "https://rakazo.example.com#space_1": { + sourceId: "source-1", + listing: { cursor: "500", token: "token-a", lastItem: last }, + changeToken: null, + uploads: {}, + }, + }, + }); + await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }); + expect(pushes).toEqual([{ items: 1, syncToken: "token-a", complete: true }]); + }); + + it("relists everything when items were deleted or the token expired", async () => { + const { helper } = fakeHelper({ size: 3 }); + const { api, pushes } = fakeApi(); + const { store } = memoryStore({ + version: 1, + deviceId: "device-1", + targets: { + "https://rakazo.example.com#space_1": { + sourceId: "source-1", + listing: null, + changeToken: "old", + uploads: {}, + }, + }, + }); + vi.mocked(helper.changes).mockResolvedValueOnce({ + token: "new", + changed: [], + deleted: ["gone/L0/001"], + fullResyncRequired: false, + }); + const result = await syncApplePhotos({ + ...baseOptions(), + helper, + api, + store, + uploadChunk: vi.fn(), + }); + expect(result.fullListing).toBe(true); + expect(pushes).toEqual([{ items: 3, syncToken: "token-3", complete: true }]); + }); + + it("registers again when the server no longer knows the source", async () => { + const { helper } = fakeHelper({ size: 2 }); + const { api } = fakeApi(); + const { store } = memoryStore({ + version: 1, + deviceId: "device-1", + targets: { + "https://rakazo.example.com#space_1": { + sourceId: "deleted-source", + listing: { cursor: null, token: null, lastItem: null }, + changeToken: null, + uploads: {}, + }, + }, + }); + vi.mocked(api.media.sources.pushItems).mockRejectedValueOnce( + Object.assign(new Error("Not found"), { code: "NOT_FOUND" }), + ); + await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }); + expect(api.media.sources.registerClient).toHaveBeenCalledTimes(1); + expect(vi.mocked(api.media.sources.pushItems).mock.calls.at(-1)?.[0].sourceId).toBe("source-1"); + }); + + it("stops before anything is sent when Photos access is denied", async () => { + const { helper } = fakeHelper({ size: 2 }); + vi.mocked(helper.authorize).mockResolvedValueOnce("denied"); + const { api } = fakeApi(); + await expect( + syncApplePhotos({ + ...baseOptions(), + helper, + api, + store: memoryStore().store, + uploadChunk: vi.fn(), + }), + ).rejects.toMatchObject({ code: "not_authorized" }); + expect(api.media.sources.registerClient).not.toHaveBeenCalled(); + }); +}); + +describe("Apple Photos original upload", () => { + const awaitingAssets = [ + asset("a1", "ASSET-00001/L0/001", "awaiting_upload"), + asset("a2", "ASSET-00002/L0/001", "proxied"), + asset("a3", "ASSET-00003/L0/001", "awaiting_upload"), + asset("a4", "ASSET-00004/L0/001", "awaiting_upload"), + ]; + + it("uploads every awaiting original in chunks with two at a time and cleans up", async () => { + const { helper, exports, stats } = fakeHelper({ size: 1 }); + const { api } = fakeApi({ assets: awaitingAssets, chunkBytes: 8 }); + const sent = new Map(); + const uploadChunk = vi.fn(async (uploadId: string, index: number, bytes: Uint8Array) => { + const parts = sent.get(uploadId) ?? []; + parts[index] = Buffer.from(bytes); + sent.set(uploadId, parts); + }); + const progress = vi.fn(); + + const result = await syncApplePhotos({ + ...baseOptions(), + helper, + api, + store: memoryStore().store, + uploadChunk, + onProgress: progress, + }); + + expect(result.uploaded).toBe(3); + expect( + vi + .mocked(helper.exportOriginal) + .mock.calls.map((call) => call[0]) + .sort(), + ).toEqual(["ASSET-00001/L0/001", "ASSET-00003/L0/001", "ASSET-00004/L0/001"]); + expect(stats.maxActiveExports).toBe(2); + expect(Buffer.concat(sent.get("upload-a1")!).toString()).toBe( + "original bytes of ASSET-00001/L0/001", + ); + expect(sent.get("upload-a1")).toHaveLength(Math.ceil(36 / 8)); + expect(api.media.uploads.begin).toHaveBeenCalledWith( + expect.objectContaining({ + assetId: "a1", + sizeBytes: 36, + sha256: sha("original bytes of ASSET-00001/L0/001"), + }), + expect.anything(), + ); + expect(api.media.uploads.complete).toHaveBeenCalledTimes(3); + expect(progress).toHaveBeenLastCalledWith({ + phase: "uploading", + indexed: 1, + uploadsDone: 3, + uploadsTotal: 3, + }); + for (const dir of exports) await expect(readdir(dir)).rejects.toThrow(); + await expect(readdir(tempRoot)).resolves.toEqual([]); + }); + + it("resumes an open upload and sends only the missing chunks", async () => { + const { helper } = fakeHelper({ size: 1 }); + const contents = "original bytes of ASSET-00001/L0/001"; + const { api } = fakeApi({ + assets: [awaitingAssets[0]!], + uploads: { + "upload-old": { + id: "upload-old", + assetId: "a1", + sizeBytes: Buffer.byteLength(contents), + chunkBytes: 10, + receivedChunks: [0, 2], + status: "open", + }, + }, + }); + const { store, current } = memoryStore({ + version: 1, + deviceId: "device-1", + targets: { + "https://rakazo.example.com#space_1": { + sourceId: "source-1", + listing: null, + changeToken: "tok", + uploads: { a1: "upload-old" }, + }, + }, + }); + vi.mocked(helper.changes).mockResolvedValueOnce({ + token: "tok", + changed: [], + deleted: [], + fullResyncRequired: false, + }); + const uploadChunk = vi.fn(async () => {}); + + await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk }); + + expect(api.media.uploads.begin).not.toHaveBeenCalled(); + expect( + uploadChunk.mock.calls.map((call) => [ + call[0], + call[1], + Buffer.from(call[2] as Uint8Array).toString(), + ]), + ).toEqual([ + ["upload-old", 1, contents.slice(10, 20)], + ["upload-old", 3, contents.slice(30)], + ]); + expect(current().targets["https://rakazo.example.com#space_1"]?.uploads).toEqual({}); + }); + + it("aborts on a checksum mismatch without starting an upload", async () => { + const { helper } = fakeHelper({ size: 1 }); + vi.mocked(helper.exportOriginal).mockImplementationOnce(async (_id, destination) => { + const file = path.join(destination, "IMG.MOV"); + await writeFile(file, "tampered"); + return { path: file, fileName: "IMG.MOV", sizeBytes: 8, sha256: sha("original") }; + }); + const { api } = fakeApi({ assets: [awaitingAssets[0]!] }); + const uploadChunk = vi.fn(); + + await expect( + syncApplePhotos({ ...baseOptions(), helper, api, store: memoryStore().store, uploadChunk }), + ).rejects.toMatchObject({ code: "integrity" }); + expect(api.media.uploads.begin).not.toHaveBeenCalled(); + expect(uploadChunk).not.toHaveBeenCalled(); + await expect(readdir(tempRoot)).resolves.toEqual([]); + }); + + it("cancels a running upload", async () => { + const { helper } = fakeHelper({ size: 1 }); + const { api } = fakeApi({ assets: [awaitingAssets[0]!], chunkBytes: 4 }); + const controller = new AbortController(); + const uploadChunk = vi.fn( + async (_id: string, index: number, _bytes: Uint8Array, signal: AbortSignal) => { + if (index === 1) controller.abort(); + signal.throwIfAborted(); + }, + ); + + await expect( + syncApplePhotos({ + ...baseOptions(), + signal: controller.signal, + helper, + api, + store: memoryStore().store, + uploadChunk, + }), + ).rejects.toMatchObject({ code: "cancelled" }); + expect(uploadChunk).toHaveBeenCalledTimes(2); + expect(api.media.uploads.complete).not.toHaveBeenCalled(); + await expect(readdir(tempRoot)).resolves.toEqual([]); + }); + + it("refuses to guess which Photos item an asset is", async () => { + const { helper } = fakeHelper({ size: 1 }); + const { api } = fakeApi({ assets: [asset("a1", undefined, "awaiting_upload")] }); + await expect( + syncApplePhotos({ + ...baseOptions(), + helper, + api, + store: memoryStore().store, + uploadChunk: vi.fn(), + }), + ).rejects.toMatchObject({ code: "unsupported_server" }); + expect(helper.exportOriginal).not.toHaveBeenCalled(); + }); +}); + +describe("Apple Photos sync state file", () => { + it("keeps a stable device id across launches", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "rakazo-photos-state-")); + try { + const store = createPhotosSyncFileStore(dir, () => "device-a"); + const first = await store.read(); + expect(first.deviceId).toBe("device-a"); + first.targets.x = { sourceId: "s", listing: null, changeToken: "t", uploads: {} }; + await store.write(first); + const reopened = await createPhotosSyncFileStore(dir, () => "device-b").read(); + expect(reopened).toEqual(first); + expect(await readFile(path.join(dir, "media-photos-sync.json"), "utf8")).toContain( + "device-a", + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it("drops unreadable state and malformed targets", () => { + expect(parsePhotosSyncState("{")).toBeNull(); + expect( + parsePhotosSyncState(JSON.stringify({ version: 2, deviceId: "d", targets: {} })), + ).toBeNull(); + expect( + parsePhotosSyncState( + JSON.stringify({ + version: 1, + deviceId: "d", + targets: { + good: { sourceId: "s", listing: null, changeToken: null, uploads: {} }, + bad: { sourceId: 4, listing: null, changeToken: null, uploads: {} }, + badListing: { sourceId: "s", listing: { cursor: 3 }, changeToken: null, uploads: {} }, + }, + }), + )?.targets, + ).toEqual({ good: { sourceId: "s", listing: null, changeToken: null, uploads: {} } }); + }); +}); + +describe("chunk uploader", () => { + it("PUTs raw bytes with the session and space", async () => { + const fetch = vi.fn(async () => new Response(null, { status: 204 })); + const upload = createChunkUploader({ + origin: "https://rakazo.example.com", + spaceId: "space_1", + fetch, + }); + const signal = new AbortController().signal; + await upload("up/1", 3, new Uint8Array([1, 2, 3]), signal); + expect(fetch).toHaveBeenCalledWith( + "https://rakazo.example.com/api/media/uploads/up%2F1/chunks/3", + expect.objectContaining({ + method: "PUT", + credentials: "include", + redirect: "error", + signal, + headers: { "content-type": "application/octet-stream", "x-rakazo-space-id": "space_1" }, + }), + ); + }); + + it("rejects failed chunks and non-origin targets", async () => { + const fetch = vi.fn(async () => new Response(null, { status: 409 })); + const upload = createChunkUploader({ + origin: "https://rakazo.example.com", + spaceId: null, + fetch, + }); + await expect( + upload("u", 0, new Uint8Array(), new AbortController().signal), + ).rejects.toMatchObject({ + code: "upload_failed", + }); + expect(() => + createChunkUploader({ origin: "https://rakazo.example.com/rpc", spaceId: null, fetch }), + ).toThrow(); + expect(() => createChunkUploader({ origin: "file:///etc", spaceId: null, fetch })).toThrow(); + }); +}); + +describe("runPool", () => { + it("bounds concurrency and reports the first failure", async () => { + let active = 0; + let max = 0; + const seen: number[] = []; + await expect( + runPool([1, 2, 3, 4, 5], 2, new AbortController().signal, async (item, signal) => { + active += 1; + max = Math.max(max, active); + await new Promise((resolve) => setTimeout(resolve, 5)); + active -= 1; + if (item === 2) throw new Error("boom"); + signal.throwIfAborted(); + seen.push(item); + }), + ).rejects.toThrow("boom"); + expect(max).toBe(2); + expect(seen).not.toContain(4); + }); +}); diff --git a/apps/desktop/src/media-photos-sync.ts b/apps/desktop/src/media-photos-sync.ts new file mode 100644 index 0000000..1c1b8c4 --- /dev/null +++ b/apps/desktop/src/media-photos-sync.ts @@ -0,0 +1,600 @@ +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { mkdtemp, open, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { ClientMediaItem, MediaAsset, MediaUpload, mediaRpcShapes } from "@rakazo/contracts"; +import type { PhotosHelper } from "./media-photos-helper.js"; +import { PHOTOS_MAX_PAGE } from "./media-photos-helper.js"; +import { readPrivateFile, writePrivateFile } from "./setup-store.js"; + +/* + * Apple Photos is a client-pushed source: the API may run on another machine, so the + * app never sends a local path. It pushes metadata in batches and uploads the originals + * the server asks for (`awaiting_upload`) in resumable raw-byte chunks. + */ + +type Shapes = typeof mediaRpcShapes; +type Shape = { input: { _input: unknown }; output: { _output: unknown } }; +/** One oRPC procedure as `ContractRouterClient` exposes it. */ +type Procedure = ( + input: S["input"]["_input"], + options?: { signal?: AbortSignal }, +) => Promise; + +/** The slice of the app RPC client the sync needs; `rpc` from a contract client satisfies it. */ +export interface MediaSyncApi { + media: { + sources: { + registerClient: Procedure; + pushItems: Procedure; + }; + assets: { search: Procedure }; + uploads: { + begin: Procedure; + status: Procedure; + complete: Procedure; + }; + }; +} + +/** Sends one raw chunk to `PUT /api/media/uploads/:uploadId/chunks/:index`. */ +export type UploadChunk = ( + uploadId: string, + index: number, + bytes: Uint8Array, + signal: AbortSignal, +) => Promise; + +export type PhotosSyncErrorCode = + | "not_authorized" + | "cancelled" + | "integrity" + | "upload_failed" + | "unsupported_server"; + +export class PhotosSyncError extends Error { + constructor( + readonly code: PhotosSyncErrorCode, + message: string, + ) { + super(message); + this.name = "PhotosSyncError"; + } +} + +// --------------------------------------------------------------------------- +// Persistent state +// --------------------------------------------------------------------------- + +export interface PhotosSyncTargetState { + sourceId: string; + /** A full listing in progress: where to resume, the token from its start and the last pushed item. */ + listing: { cursor: string | null; token: string | null; lastItem: ClientMediaItem | null } | null; + /** Set after a complete listing; later syncs only read changes since it. */ + changeToken: string | null; + /** Open uploads by asset id so an interrupted upload resumes instead of restarting. */ + uploads: Record; +} + +export interface PhotosSyncState { + version: 1; + /** Stable per installation; the server keys the Apple Photos source by it. */ + deviceId: string; + /** Keyed by server origin and space, since each has its own source. */ + targets: Record; +} + +export interface PhotosSyncStore { + read(): Promise; + write(state: PhotosSyncState): Promise; +} + +export const PHOTOS_SYNC_FILE_NAME = "media-photos-sync.json"; +const MAX_STATE_BYTES = 4 * 1024 * 1024; + +/** Kept next to the desktop setup file, private to the user and written atomically. */ +export function createPhotosSyncFileStore( + userDataDir: string, + newDeviceId: () => string = randomUUID, +): PhotosSyncStore { + const file = path.join(userDataDir, PHOTOS_SYNC_FILE_NAME); + return { + async read() { + const raw = await readPrivateFile(file, MAX_STATE_BYTES); + return (raw === null ? null : parsePhotosSyncState(raw)) ?? emptyState(newDeviceId()); + }, + write: (state) => writePrivateFile(file, `${JSON.stringify(state)}\n`), + }; +} + +function emptyState(deviceId: string): PhotosSyncState { + return { version: 1, deviceId, targets: {} }; +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); +const isNullableString = (value: unknown) => value === null || typeof value === "string"; + +/** Unreadable state starts over: the server deduplicates by device and external id. */ +export function parsePhotosSyncState(raw: string): PhotosSyncState | null { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + return null; + } + if ( + !isRecord(value) || + value.version !== 1 || + typeof value.deviceId !== "string" || + value.deviceId === "" || + value.deviceId.length > 200 || + !isRecord(value.targets) + ) { + return null; + } + const targets: Record = {}; + for (const [key, target] of Object.entries(value.targets)) { + if ( + !isRecord(target) || + typeof target.sourceId !== "string" || + !isNullableString(target.changeToken) || + !isRecord(target.uploads) || + !Object.values(target.uploads).every((id) => typeof id === "string") + ) { + continue; + } + const listing = target.listing; + const validListing = + listing === null || + (isRecord(listing) && + isNullableString(listing.cursor) && + isNullableString(listing.token) && + (listing.lastItem === null || isRecord(listing.lastItem))); + if (!validListing) continue; + targets[key] = target as unknown as PhotosSyncTargetState; + } + return { version: 1, deviceId: value.deviceId, targets }; +} + +// --------------------------------------------------------------------------- +// Chunk upload over HTTP +// --------------------------------------------------------------------------- + +/** + * Chunk PUTs ride the same signed-in session as the app window: the injected `fetch` + * must carry that session's cookies (Electron `session.fetch` with credentials). + */ +export function createChunkUploader(options: { + origin: string; + spaceId: string | null; + fetch: (url: string, init: RequestInit) => Promise; +}): UploadChunk { + const origin = new URL(options.origin); + if (!["http:", "https:"].includes(origin.protocol) || origin.origin !== options.origin) { + throw new Error("Uploads need the server origin."); + } + return async (uploadId, index, bytes, signal) => { + if (!Number.isSafeInteger(index) || index < 0) throw new Error("Invalid chunk index."); + const headers: Record = { "content-type": "application/octet-stream" }; + if (options.spaceId) headers["x-rakazo-space-id"] = options.spaceId; + const response = await options.fetch( + `${origin.origin}/api/media/uploads/${encodeURIComponent(uploadId)}/chunks/${index}`, + { + method: "PUT", + headers, + body: bytes, + credentials: "include", + redirect: "error", + signal, + }, + ); + if (!response.ok) { + throw new PhotosSyncError( + "upload_failed", + `The server rejected an upload (${response.status}).`, + ); + } + }; +} + +// --------------------------------------------------------------------------- +// Sync +// --------------------------------------------------------------------------- + +export interface PhotosSyncProgress { + phase: "authorizing" | "indexing" | "uploading"; + /** Items pushed in this run. */ + indexed: number; + uploadsDone: number; + uploadsTotal: number; +} + +export interface PhotosSyncResult { + indexed: number; + created: number; + updated: number; + unchanged: number; + uploaded: number; + /** True when this run listed the whole library instead of reading changes. */ + fullListing: boolean; +} + +export interface PhotosSyncOptions { + helper: PhotosHelper; + api: MediaSyncApi; + uploadChunk: UploadChunk; + store: PhotosSyncStore; + /** Server origin plus space, e.g. `https://rakazo.example.com#space_1`. */ + target: string; + /** Source label shown in the library, e.g. the Mac's name or "Apple Photos". */ + label: string; + /** Exports land in a fresh folder below this and are deleted after upload. */ + tempRoot: string; + signal: AbortSignal; + onProgress?: (progress: PhotosSyncProgress) => void; + /** At most 500, the contract's batch limit. */ + batchSize?: number; + concurrency?: number; +} + +const MAX_SYNC_TOKEN_LENGTH = 2000; + +export async function syncApplePhotos(options: PhotosSyncOptions): Promise { + const { helper, api, signal } = options; + const batchSize = Math.min(Math.max(1, options.batchSize ?? PHOTOS_MAX_PAGE), PHOTOS_MAX_PAGE); + const result: PhotosSyncResult = { + indexed: 0, + created: 0, + updated: 0, + unchanged: 0, + uploaded: 0, + fullListing: false, + }; + const progress = { uploadsDone: 0, uploadsTotal: 0 }; + const report = (phase: PhotosSyncProgress["phase"]) => + options.onProgress?.({ phase, indexed: result.indexed, ...progress }); + + const state = await options.store.read(); + // Writes are serialized so concurrent upload workers never interleave partial state. + let saving = Promise.resolve(); + const save = () => { + const snapshot = JSON.parse(JSON.stringify(state)) as PhotosSyncState; + saving = saving.then(() => options.store.write(snapshot)); + return saving; + }; + + try { + report("authorizing"); + const authorization = await helper.authorize(true, signal); + if (authorization !== "authorized" && authorization !== "limited") { + throw new PhotosSyncError("not_authorized", "Photos access was not granted."); + } + + const ensureTarget = async (): Promise => { + const existing = state.targets[options.target]; + if (existing) return existing; + const source = await api.media.sources.registerClient( + { kind: "apple_photos", label: options.label, deviceId: state.deviceId }, + { signal }, + ); + const created: PhotosSyncTargetState = { + sourceId: source.id, + listing: null, + changeToken: null, + uploads: {}, + }; + state.targets[options.target] = created; + await save(); + return created; + }; + + const push = async ( + target: PhotosSyncTargetState, + items: ClientMediaItem[], + syncToken: string | null, + complete: boolean, + ) => { + const counts = await api.media.sources.pushItems( + { + sourceId: target.sourceId, + items, + syncToken: + syncToken !== null && syncToken.length <= MAX_SYNC_TOKEN_LENGTH ? syncToken : null, + complete, + }, + { signal }, + ); + result.indexed += items.length; + result.created += counts.created; + result.updated += counts.updated; + result.unchanged += counts.unchanged; + report("indexing"); + }; + + /** Full listing, resumable page by page; the last push says `complete`. */ + const listAll = async (target: PhotosSyncTargetState) => { + result.fullListing = true; + target.listing ??= { cursor: null, token: null, lastItem: null }; + const listing = target.listing; + while (true) { + const page = await helper.list({ cursor: listing.cursor, limit: batchSize }, signal); + if (listing.cursor === null) listing.token = page.changeToken; + const done = page.nextCursor === null; + // An empty final page (deletions while resuming) still has to close the listing, + // and pushItems needs at least one item: resend the last one, which is a no-op. + const items = + page.items.length > 0 || !done || listing.lastItem === null + ? page.items + : [listing.lastItem]; + if (items.length > 0) await push(target, items, done ? listing.token : null, done); + if (done) { + target.changeToken = listing.token; + target.listing = null; + await save(); + return; + } + listing.cursor = page.nextCursor; + listing.lastItem = items.at(-1) ?? listing.lastItem; + await save(); + } + }; + + const syncMetadata = async () => { + const target = await ensureTarget(); + if (target.listing !== null || target.changeToken === null) { + await listAll(target); + return; + } + const changes = await helper.changes(target.changeToken, signal); + // pushItems cannot express deletions; a complete listing lets the server reconcile them. + if (changes.fullResyncRequired || changes.deleted.length > 0 || changes.token === null) { + target.listing = null; + await listAll(target); + return; + } + for (let start = 0; start < changes.changed.length; start += batchSize) { + const batch = changes.changed.slice(start, start + batchSize); + const last = start + batchSize >= changes.changed.length; + await push(target, batch, last ? changes.token : null, false); + } + target.changeToken = changes.token; + await save(); + }; + + report("indexing"); + try { + await syncMetadata(); + } catch (error) { + // The source was removed on the server: register again and list from scratch. + if (!isNotFound(error) || signal.aborted) throw error; + delete state.targets[options.target]; + await save(); + await syncMetadata(); + } + + const target = state.targets[options.target]!; + const awaiting = await awaitingUploads(api, target.sourceId, signal); + progress.uploadsTotal = awaiting.length; + // Forget uploads for assets the server no longer waits for. + const waiting = new Set(awaiting.map((asset) => asset.id)); + for (const assetId of Object.keys(target.uploads)) { + if (!waiting.has(assetId)) delete target.uploads[assetId]; + } + await save(); + report("uploading"); + await runPool(awaiting, options.concurrency ?? 2, signal, async (asset, workerSignal) => { + await uploadAsset({ ...options, signal: workerSignal }, target, asset, save); + result.uploaded += 1; + progress.uploadsDone += 1; + report("uploading"); + }); + return result; + } catch (error) { + if (signal.aborted) throw new PhotosSyncError("cancelled", "Sync cancelled."); + throw error; + } finally { + await saving.catch(() => undefined); + } +} + +/** + * Binds the sync to the server and space the request names. `connect` builds the RPC + * client and chunk uploader on the signed-in session for that space. + */ +export function createPhotosSyncRunner(deps: { + helper: PhotosHelper; + store: PhotosSyncStore; + tempRoot: string; + label: string; + connect: ( + spaceId: string | null, + ) => Promise<{ origin: string; api: MediaSyncApi; uploadChunk: UploadChunk }>; +}) { + return async (request: { + spaceId: string | null; + signal: AbortSignal; + onProgress: (progress: PhotosSyncProgress) => void; + }): Promise => { + const { origin, api, uploadChunk } = await deps.connect(request.spaceId); + return syncApplePhotos({ + helper: deps.helper, + store: deps.store, + tempRoot: deps.tempRoot, + label: deps.label, + api, + uploadChunk, + target: `${origin}#${request.spaceId ?? "default"}`, + signal: request.signal, + onProgress: request.onProgress, + }); + }; +} + +function isNotFound(error: unknown): boolean { + return isRecord(error) && error.code === "NOT_FOUND"; +} + +interface AwaitingAsset { + id: string; + externalId: string; +} + +/** + * The contract has no "awaiting upload" query, so this pages through the source's + * assets and filters on `ingestState`. Mapping an asset back to a Photos item needs its + * `externalId`, which `MediaAssetSchema` does not expose yet; without it uploads fail + * loudly instead of guessing. + */ +export async function awaitingUploads( + api: MediaSyncApi, + sourceId: string, + signal: AbortSignal, +): Promise { + const awaiting: AwaitingAsset[] = []; + let cursor: string | null = null; + do { + const page: { items: MediaAsset[]; nextCursor: string | null } = await api.media.assets.search( + { sourceId, cursor, limit: PHOTOS_MAX_PAGE }, + { signal }, + ); + for (const asset of page.items) { + if (asset.ingestState !== "awaiting_upload" || asset.sourceId !== sourceId) continue; + const externalId = (asset as MediaAsset & { externalId?: unknown }).externalId; + if (typeof externalId !== "string" || externalId === "") { + throw new PhotosSyncError( + "unsupported_server", + "The server does not say which Photos item an asset is.", + ); + } + awaiting.push({ id: asset.id, externalId }); + } + cursor = page.nextCursor; + } while (cursor !== null); + return awaiting; +} + +async function uploadAsset( + options: PhotosSyncOptions, + target: PhotosSyncTargetState, + asset: AwaitingAsset, + save: () => Promise, +): Promise { + const { api, signal } = options; + const directory = await mkdtemp(path.join(options.tempRoot, "rakazo-photos-")); + try { + const exported = await options.helper.exportOriginal(asset.externalId, directory, { signal }); + // The helper's hash is checked against the bytes on disk before anything is sent. + const local = await hashFile(exported.path, signal); + if (local.sha256 !== exported.sha256 || local.sizeBytes !== exported.sizeBytes) { + throw new PhotosSyncError("integrity", "The exported file does not match its checksum."); + } + + let upload: MediaUpload | null = null; + const previous = target.uploads[asset.id]; + if (previous) { + const status = await api.media.uploads + .status({ uploadId: previous }, { signal }) + .catch((error: unknown) => { + if (isNotFound(error)) return null; + throw error; + }); + if (status?.status === "complete") { + delete target.uploads[asset.id]; + await save(); + return; + } + if (status?.status === "open" && status.sizeBytes === local.sizeBytes) upload = status; + } + if (upload === null) { + upload = await api.media.uploads.begin( + { + assetId: asset.id, + sizeBytes: local.sizeBytes, + sha256: local.sha256, + fileName: exported.fileName.slice(0, 255), + }, + { signal }, + ); + target.uploads[asset.id] = upload.id; + await save(); + } + + const received = new Set(upload.receivedChunks); + const chunkCount = Math.ceil(local.sizeBytes / upload.chunkBytes); + const file = await open(exported.path, "r"); + try { + for (let index = 0; index < chunkCount; index += 1) { + if (received.has(index)) continue; + signal.throwIfAborted(); + const offset = index * upload.chunkBytes; + const length = Math.min(upload.chunkBytes, local.sizeBytes - offset); + const bytes = Buffer.alloc(length); + const { bytesRead } = await file.read(bytes, 0, length, offset); + if (bytesRead !== length) { + throw new PhotosSyncError("integrity", "The exported file changed during upload."); + } + await options.uploadChunk(upload.id, index, bytes, signal); + } + } finally { + await file.close(); + } + const completed = await api.media.uploads.complete({ uploadId: upload.id }, { signal }); + if (completed.status !== "complete") { + throw new PhotosSyncError("upload_failed", "The server did not accept the upload."); + } + delete target.uploads[asset.id]; + await save(); + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +export async function hashFile( + file: string, + signal?: AbortSignal, +): Promise<{ sha256: string; sizeBytes: number }> { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(file, { signal })) hash.update(chunk as Buffer); + return { sha256: hash.digest("hex"), sizeBytes: (await stat(file)).size }; +} + +/** + * Runs `work` with at most `limit` items in flight. The first failure aborts the + * others and is the error reported. + */ +export async function runPool( + items: T[], + limit: number, + signal: AbortSignal, + work: (item: T, signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + const forward = () => controller.abort(signal.reason); + signal.addEventListener("abort", forward, { once: true }); + if (signal.aborted) forward(); + let next = 0; + let failed = false; + let failure: unknown; + const worker = async () => { + while (!failed && !controller.signal.aborted && next < items.length) { + const item = items[next++]!; + try { + await work(item, controller.signal); + } catch (error) { + if (!failed) { + failed = true; + failure = error; + controller.abort(error); + } + } + } + }; + try { + await Promise.all(Array.from({ length: Math.min(Math.max(1, limit), items.length) }, worker)); + } finally { + signal.removeEventListener("abort", forward); + } + if (failed) throw failure; + signal.throwIfAborted(); +} From 0ee6c664d27a55df5d3de7440382c698b2fdaa1e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 17:57:02 +0000 Subject: [PATCH 10/16] build(db): depend on @rakazo/media for coordinate rounding Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- packages/db/package.json | 7 ++++--- pnpm-lock.yaml | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/db/package.json b/packages/db/package.json index 3f96343..96f90aa 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -18,14 +18,15 @@ "test": "vitest run --root ../.. packages/db/src" }, "dependencies": { - "@rakazo/adapter-kit": "workspace:*", "@prisma/adapter-pg": "7.10.0", "@prisma/client": "7.10.0", + "@rakazo/adapter-kit": "workspace:*", "@rakazo/contracts": "workspace:*", "@rakazo/core": "workspace:*", "@rakazo/logging": "workspace:*", - "pg": "8.23.0", - "dotenv": "17.4.2" + "@rakazo/media": "workspace:*", + "dotenv": "17.4.2", + "pg": "8.23.0" }, "devDependencies": { "@types/pg": "8.23.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 179bf2a..51dbf6b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -782,6 +782,9 @@ importers: '@rakazo/logging': specifier: workspace:* version: link:../logging + '@rakazo/media': + specifier: workspace:* + version: link:../media dotenv: specifier: 17.4.2 version: 17.4.2 From 1aab5ab3a82f0997afee7434080469a30a8f9615 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:00:02 +0000 Subject: [PATCH 11/16] feat(media): expose asset externalId, filter by ingest state, push device deletions The desktop Photos sync needs to map awaiting assets back to Photos items, query only awaiting uploads, and report deletions without a full relisting. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- apps/desktop/src/media-photos-sync.test.ts | 47 ++++++++---------- apps/desktop/src/media-photos-sync.ts | 58 +++++++++------------- packages/contracts/src/media.ts | 26 +++++++--- packages/db/src/media-library.ts | 44 ++++++++++++++++ packages/db/src/media-shared.ts | 1 + packages/db/src/media.postgres.test.ts | 27 ++++++++++ packages/db/src/media.test.ts | 1 + 7 files changed, 138 insertions(+), 66 deletions(-) diff --git a/apps/desktop/src/media-photos-sync.test.ts b/apps/desktop/src/media-photos-sync.test.ts index a1928a8..d37b569 100644 --- a/apps/desktop/src/media-photos-sync.test.ts +++ b/apps/desktop/src/media-photos-sync.test.ts @@ -88,10 +88,11 @@ function memoryStore(initial?: PhotosSyncState) { return { store, writes, current: () => state }; } -function asset(id: string, externalId: string | undefined, ingestState: MediaAsset["ingestState"]) { +function asset(id: string, externalId: string, ingestState: MediaAsset["ingestState"]) { return { id, sourceId: "source-1", + externalId, kind: "video", storage: "remote", capturedAt: null, @@ -109,7 +110,6 @@ function asset(id: string, externalId: string | undefined, ingestState: MediaAss hasThumbnail: false, qualityScore: null, createdAt: "2024-01-01T00:00:00.000Z", - ...(externalId === undefined ? {} : { externalId }), } as MediaAsset; } @@ -121,7 +121,12 @@ function fakeApi( } = {}, ) { const uploads = new Map(Object.entries(options.uploads ?? {})); - const pushes: { items: number; syncToken: string | null; complete: boolean }[] = []; + const pushes: { + items: number; + syncToken: string | null; + complete: boolean; + deleted?: number; + }[] = []; const chunkBytes = options.chunkBytes ?? 4; const api = { media: { @@ -138,15 +143,22 @@ function fakeApi( deviceId: input.deviceId, })), pushItems: vi.fn( - async (input: { items: unknown[]; syncToken: string | null; complete: boolean }) => { - expect(input.items.length).toBeGreaterThan(0); + async (input: { + items: unknown[]; + deletedExternalIds?: string[]; + syncToken: string | null; + complete: boolean; + }) => { + const deleted = input.deletedExternalIds?.length ?? 0; + expect(input.items.length > 0 || deleted > 0 || input.complete).toBe(true); expect(input.items.length).toBeLessThanOrEqual(500); pushes.push({ items: input.items.length, syncToken: input.syncToken, complete: input.complete, + ...(deleted > 0 ? { deleted } : {}), }); - return { created: input.items.length, updated: 0, unchanged: 0 }; + return { created: input.items.length, updated: 0, unchanged: 0, deleted }; }, ), }, @@ -309,10 +321,10 @@ describe("Apple Photos metadata sync", () => { }, }); await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }); - expect(pushes).toEqual([{ items: 1, syncToken: "token-a", complete: true }]); + expect(pushes).toEqual([{ items: 0, syncToken: "token-a", complete: true }]); }); - it("relists everything when items were deleted or the token expired", async () => { + it("pushes device deletions without relisting", async () => { const { helper } = fakeHelper({ size: 3 }); const { api, pushes } = fakeApi(); const { store } = memoryStore({ @@ -340,8 +352,8 @@ describe("Apple Photos metadata sync", () => { store, uploadChunk: vi.fn(), }); - expect(result.fullListing).toBe(true); - expect(pushes).toEqual([{ items: 3, syncToken: "token-3", complete: true }]); + expect(result.fullListing).toBe(false); + expect(pushes).toEqual([{ items: 0, syncToken: "new", complete: false, deleted: 1 }]); }); it("registers again when the server no longer knows the source", async () => { @@ -538,21 +550,6 @@ describe("Apple Photos original upload", () => { expect(api.media.uploads.complete).not.toHaveBeenCalled(); await expect(readdir(tempRoot)).resolves.toEqual([]); }); - - it("refuses to guess which Photos item an asset is", async () => { - const { helper } = fakeHelper({ size: 1 }); - const { api } = fakeApi({ assets: [asset("a1", undefined, "awaiting_upload")] }); - await expect( - syncApplePhotos({ - ...baseOptions(), - helper, - api, - store: memoryStore().store, - uploadChunk: vi.fn(), - }), - ).rejects.toMatchObject({ code: "unsupported_server" }); - expect(helper.exportOriginal).not.toHaveBeenCalled(); - }); }); describe("Apple Photos sync state file", () => { diff --git a/apps/desktop/src/media-photos-sync.ts b/apps/desktop/src/media-photos-sync.ts index 1c1b8c4..72ea3f9 100644 --- a/apps/desktop/src/media-photos-sync.ts +++ b/apps/desktop/src/media-photos-sync.ts @@ -45,12 +45,7 @@ export type UploadChunk = ( signal: AbortSignal, ) => Promise; -export type PhotosSyncErrorCode = - | "not_authorized" - | "cancelled" - | "integrity" - | "upload_failed" - | "unsupported_server"; +export type PhotosSyncErrorCode = "not_authorized" | "cancelled" | "integrity" | "upload_failed"; export class PhotosSyncError extends Error { constructor( @@ -294,11 +289,13 @@ export async function syncApplePhotos(options: PhotosSyncOptions): Promise { const counts = await api.media.sources.pushItems( { sourceId: target.sourceId, items, + deletedExternalIds, syncToken: syncToken !== null && syncToken.length <= MAX_SYNC_TOKEN_LENGTH ? syncToken : null, complete, @@ -321,13 +318,9 @@ export async function syncApplePhotos(options: PhotosSyncOptions): Promise 0 || !done || listing.lastItem === null - ? page.items - : [listing.lastItem]; - if (items.length > 0) await push(target, items, done ? listing.token : null, done); + const items = page.items; + // An empty final page still closes the listing. + if (items.length > 0 || done) await push(target, items, done ? listing.token : null, done); if (done) { target.changeToken = listing.token; target.listing = null; @@ -347,16 +340,23 @@ export async function syncApplePhotos(options: PhotosSyncOptions): Promise 0 || changes.token === null) { + if (changes.fullResyncRequired || changes.token === null) { target.listing = null; await listAll(target); return; } - for (let start = 0; start < changes.changed.length; start += batchSize) { - const batch = changes.changed.slice(start, start + batchSize); - const last = start + batchSize >= changes.changed.length; - await push(target, batch, last ? changes.token : null, false); + const batches = Math.max( + Math.ceil(changes.changed.length / batchSize), + Math.ceil(changes.deleted.length / DELETED_BATCH_SIZE), + ); + for (let index = 0; index < batches; index++) { + const batch = changes.changed.slice(index * batchSize, (index + 1) * batchSize); + const deleted = changes.deleted.slice( + index * DELETED_BATCH_SIZE, + (index + 1) * DELETED_BATCH_SIZE, + ); + const last = index === batches - 1; + await push(target, batch, last ? changes.token : null, false, deleted); } target.changeToken = changes.token; await save(); @@ -440,12 +440,9 @@ interface AwaitingAsset { externalId: string; } -/** - * The contract has no "awaiting upload" query, so this pages through the source's - * assets and filters on `ingestState`. Mapping an asset back to a Photos item needs its - * `externalId`, which `MediaAssetSchema` does not expose yet; without it uploads fail - * loudly instead of guessing. - */ +/** Assets of this source the server is waiting for, mapped back to Photos items. */ +const DELETED_BATCH_SIZE = 5000; + export async function awaitingUploads( api: MediaSyncApi, sourceId: string, @@ -455,19 +452,12 @@ export async function awaitingUploads( let cursor: string | null = null; do { const page: { items: MediaAsset[]; nextCursor: string | null } = await api.media.assets.search( - { sourceId, cursor, limit: PHOTOS_MAX_PAGE }, + { sourceId, ingestStates: ["awaiting_upload"], cursor, limit: PHOTOS_MAX_PAGE }, { signal }, ); for (const asset of page.items) { if (asset.ingestState !== "awaiting_upload" || asset.sourceId !== sourceId) continue; - const externalId = (asset as MediaAsset & { externalId?: unknown }).externalId; - if (typeof externalId !== "string" || externalId === "") { - throw new PhotosSyncError( - "unsupported_server", - "The server does not say which Photos item an asset is.", - ); - } - awaiting.push({ id: asset.id, externalId }); + awaiting.push({ id: asset.id, externalId: asset.externalId }); } cursor = page.nextCursor; } while (cursor !== null); diff --git a/packages/contracts/src/media.ts b/packages/contracts/src/media.ts index 428fa44..b70e5b8 100644 --- a/packages/contracts/src/media.ts +++ b/packages/contracts/src/media.ts @@ -735,6 +735,8 @@ export type MediaSource = z.infer; export const MediaAssetSchema = z.object({ id: Id, sourceId: Id, + /** Identifier inside the source (relative path, Photos localIdentifier). */ + externalId: z.string(), kind: MediaAssetKindSchema, storage: MediaAssetStorageSchema, capturedAt: IsoDate.nullable(), @@ -1090,6 +1092,7 @@ export const MediaAssetQuerySchema = PageInput.extend({ personId: Id.optional(), collectionId: Id.optional(), sourceId: Id.optional(), + ingestStates: z.array(MediaIngestStateSchema).max(7).optional(), minQuality: z.number().min(0).max(1).optional(), favoritesOnly: z.boolean().optional(), }); @@ -1161,17 +1164,26 @@ export const mediaRpcShapes = { output: MediaSourceSchema, }, pushItems: { - input: z.object({ - sourceId: Id, - items: z.array(ClientMediaItemSchema).min(1).max(500), - /** Opaque change token from the device for the next incremental push. */ - syncToken: z.string().max(2000).nullable(), - complete: z.boolean(), - }), + input: z + .object({ + sourceId: Id, + items: z.array(ClientMediaItemSchema).max(500), + /** Items removed on the device since the last push. */ + deletedExternalIds: z.array(z.string().min(1).max(200)).max(5000).default([]), + /** Opaque change token from the device for the next incremental push. */ + syncToken: z.string().max(2000).nullable(), + complete: z.boolean(), + }) + .refine( + (input) => + input.items.length > 0 || input.deletedExternalIds.length > 0 || input.complete, + { message: "Push items, deletions, or completion", path: ["items"] }, + ), output: z.object({ created: z.number().int(), updated: z.number().int(), unchanged: z.number().int(), + deleted: z.number().int(), }), }, sync: { input: z.object({ sourceId: Id }), output: ok }, diff --git a/packages/db/src/media-library.ts b/packages/db/src/media-library.ts index 3bee8a5..40f669b 100644 --- a/packages/db/src/media-library.ts +++ b/packages/db/src/media-library.ts @@ -307,6 +307,46 @@ export function createMediaLibraryRepos(prisma: PrismaClient) { if (result.count !== 1) throw new IsolationError(); } + /** + * Delete assets a device reported as removed. Returns the media-root-relative files + * the caller must delete (originals, proxies, thumbnails, face keyframes, masks). + */ + async function removeSourceItems( + spaceId: string, + sourceId: string, + externalIds: string[], + ): Promise<{ deleted: number; paths: string[] }> { + if (externalIds.length === 0) return { deleted: 0, paths: [] }; + return prisma.$transaction(async (tx) => { + const source = await tx.mediaSource.findFirst({ where: { id: sourceId, spaceId } }); + if (!source) throw new IsolationError(); + const assets = await tx.mediaAsset.findMany({ + where: { spaceId, sourceId, externalId: { in: externalIds } }, + select: { + id: true, + storagePath: true, + proxyPath: true, + thumbPath: true, + spritePath: true, + faceTracks: { select: { keyframesPath: true } }, + marks: { select: { maskDir: true } }, + }, + }); + const paths = assets.flatMap((asset) => [ + ...[asset.storagePath, asset.proxyPath, asset.thumbPath, asset.spritePath], + ...asset.faceTracks.map((track) => track.keyframesPath), + ...asset.marks.map((mark) => mark.maskDir), + ]); + const result = await tx.mediaAsset.deleteMany({ + where: { spaceId, id: { in: assets.map((asset) => asset.id) } }, + }); + return { + deleted: result.count, + paths: paths.filter((path): path is string => typeof path === "string" && path !== ""), + }; + }); + } + /** * Worker entry point: jobs carry only a record id. Returns the owning space so every * further call stays scoped, or null when the record no longer exists. @@ -413,6 +453,9 @@ export function createMediaLibraryRepos(prisma: PrismaClient) { if (query.kinds && query.kinds.length > 0) { conditions.push(Prisma.sql`a.kind IN (${Prisma.join(query.kinds)})`); } + if (query.ingestStates && query.ingestStates.length > 0) { + conditions.push(Prisma.sql`a."ingestState" IN (${Prisma.join(query.ingestStates)})`); + } if (query.from) conditions.push(Prisma.sql`a."capturedAt" >= ${new Date(query.from)}`); if (query.to) conditions.push(Prisma.sql`a."capturedAt" <= ${new Date(query.to)}`); if (query.favoritesOnly) conditions.push(Prisma.sql`a."isFavorite" = TRUE`); @@ -734,6 +777,7 @@ export function createMediaLibraryRepos(prisma: PrismaClient) { getSource, updateSourceSync, removeSource, + removeSourceItems, mediaSpaceOf, upsertSourceItems, searchAssets, diff --git a/packages/db/src/media-shared.ts b/packages/db/src/media-shared.ts index 589160e..b464ca3 100644 --- a/packages/db/src/media-shared.ts +++ b/packages/db/src/media-shared.ts @@ -276,6 +276,7 @@ export function mapMediaAsset(row: MediaAssetRow): MediaAsset { return { id: row.id, sourceId: row.sourceId, + externalId: row.externalId, kind: row.kind as MediaAssetKind, storage: row.storage as MediaAssetStorage, capturedAt: iso(row.capturedAt), diff --git a/packages/db/src/media.postgres.test.ts b/packages/db/src/media.postgres.test.ts index c1e233e..b82ca83 100644 --- a/packages/db/src/media.postgres.test.ts +++ b/packages/db/src/media.postgres.test.ts @@ -34,6 +34,33 @@ describePostgres("media library and organization (PostgreSQL)", () => { return { ...user, sourceId: source.id }; } + it("removes device-deleted items and filters by ingest state", async () => { + const { spaceId, sourceId } = await newLibrary("ios_library"); + const other = await newLibrary("ios_library"); + const items = Array.from({ length: 5 }, (_, index) => sourceItem(index)); + await media.upsertSourceItems(spaceId, sourceId, items); + await media.upsertSourceItems(other.spaceId, other.sourceId, items); + + const awaiting = await media.searchAssets(spaceId, { ingestStates: ["awaiting_upload"] }); + expect(awaiting.items).toHaveLength(5); + expect(awaiting.items.map((asset) => asset.externalId).sort()).toEqual( + items.map((item) => item.externalId).sort(), + ); + expect((await media.searchAssets(spaceId, { ingestStates: ["analyzed"] })).items).toEqual([]); + + const removed = await media.removeSourceItems(spaceId, sourceId, [ + items[0]!.externalId, + items[1]!.externalId, + "unknown", + ]); + expect(removed.deleted).toBe(2); + expect((await media.searchAssets(spaceId, {})).items).toHaveLength(3); + expect((await media.searchAssets(other.spaceId, {})).items).toHaveLength(5); + await expect( + media.removeSourceItems(other.spaceId, sourceId, [items[2]!.externalId]), + ).rejects.toBeInstanceOf(IsolationError); + }); + it("indexes source items once and reports only real changes on the next sync", async () => { const { spaceId, sourceId } = await newLibrary(); const items = Array.from({ length: 1000 }, (_, index) => diff --git a/packages/db/src/media.test.ts b/packages/db/src/media.test.ts index c859369..6b3c3cd 100644 --- a/packages/db/src/media.test.ts +++ b/packages/db/src/media.test.ts @@ -122,6 +122,7 @@ describe("media DTO mappers", () => { expect(dto).toEqual({ id: "asset-1", sourceId: "source-1", + externalId: asset.externalId, kind: "video", storage: "referenced", capturedAt: at.toISOString(), From 26d95a374eac0a5db82b7c4d7bf59450393d5e29 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:06:02 +0000 Subject: [PATCH 12/16] feat(media): implement pure domain logic for Media Studio Session and theme clustering, EDL validation and timeline mapping, the EDL-to-ffmpeg compiler with caption files and filtergraph escaping, teaser selection and learning, the compliance gate with compliance-only auto-fix, and the marketing helpers (UTM, hashtags, CSV metrics, intro proposals, brand-kit checks, upload checklist). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- packages/media/src/clustering.ts | 292 +++++- packages/media/src/compile.ts | 1332 ++++++++++++++++++++++++- packages/media/src/edl.ts | 411 +++++++- packages/media/src/gate.ts | 650 +++++++++++- packages/media/src/marketing.ts | 510 +++++++++- packages/media/src/not-implemented.ts | 4 - packages/media/src/teaser.ts | 599 ++++++++++- 7 files changed, 3663 insertions(+), 135 deletions(-) delete mode 100644 packages/media/src/not-implemented.ts diff --git a/packages/media/src/clustering.ts b/packages/media/src/clustering.ts index 59b696e..31bf572 100644 --- a/packages/media/src/clustering.ts +++ b/packages/media/src/clustering.ts @@ -1,5 +1,5 @@ import type { MediaAssetKind } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { cosineSimilarity, haversineKm, roundCoordinate } from "./geo.js"; export interface ClusterAsset { id: string; @@ -46,15 +46,157 @@ export interface SessionCluster { centroid: { lat: number; lon: number } | null; } +const MINUTE_MS = 60_000; +const HOUR_MS = 60 * MINUTE_MS; + +const dayFormatters = new Map(); + +/** Calendar day (YYYY-MM-DD) of an instant in an IANA time zone. */ +export function calendarDay(date: Date, timeZone: string): string { + let formatter = dayFormatters.get(timeZone); + if (!formatter) { + formatter = new Intl.DateTimeFormat("en-CA", { + timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + }); + dayFormatters.set(timeZone, formatter); + } + return formatter.format(date); +} + +type DatedAsset = ClusterAsset & { capturedAt: Date }; +type PositionedAsset = ClusterAsset & { gpsLat: number; gpsLon: number }; + +function hasPosition(asset: ClusterAsset): asset is PositionedAsset { + return ( + asset.gpsLat !== null && + asset.gpsLon !== null && + Number.isFinite(asset.gpsLat) && + Number.isFinite(asset.gpsLon) + ); +} + +function isDated(asset: ClusterAsset): asset is DatedAsset { + return asset.capturedAt !== null && Number.isFinite(asset.capturedAt.getTime()); +} + +/** Mean of the given vectors, or null when none are present or dimensions disagree. */ +export function meanVector(vectors: ArrayLike[]): Float32Array | null { + const first = vectors[0]; + if (!first) return null; + const sum = new Float64Array(first.length); + for (const vector of vectors) { + if (vector.length !== first.length) return null; + for (let i = 0; i < vector.length; i++) sum[i] = sum[i]! + (vector[i] as number); + } + const mean = new Float32Array(first.length); + for (let i = 0; i < sum.length; i++) mean[i] = sum[i]! / vectors.length; + return mean; +} + +function sessionEmbedding(assets: ClusterAsset[]): Float32Array | null { + return meanVector(assets.flatMap((asset) => (asset.embedding ? [asset.embedding] : []))); +} + +function finishSession(assets: ClusterAsset[], key: string): SessionCluster { + const dated = assets.filter(isDated); + const positioned = assets.filter(hasPosition); + const mean = (values: number[]) => values.reduce((sum, v) => sum + v, 0) / values.length; + return { + key, + assetIds: assets.map((asset) => asset.id), + startAt: dated[0]?.capturedAt ?? null, + endAt: dated.at(-1)?.capturedAt ?? null, + centroid: + positioned.length === 0 + ? null + : { + lat: roundCoordinate(mean(positioned.map((a) => a.gpsLat))), + lon: roundCoordinate(mean(positioned.map((a) => a.gpsLon))), + }, + }; +} + /** * Deterministic session grouping (spec 4.4). Assets without capturedAt form * one trailing "undated" session; exact duplicates (same id) are ignored. + * + * A new session starts on a gap longer than gapMinutes, a jump of more than + * maxDistanceKm from the session's last known position, or a calendar-day + * change (in timeZone) with a gap longer than dayBreakHours. Adjacent sessions + * are then merged when their mean embeddings reach mergeSimilarity and the gap + * between them stays below mergeMaxGapHours. Keys derive from the earliest + * asset id, so they survive re-clustering as long as that asset stays first. */ export function buildSessions( - _assets: ClusterAsset[], - _options?: Partial, + assets: ClusterAsset[], + options?: Partial, ): SessionCluster[] { - return notImplemented("buildSessions"); + const opts = { ...DEFAULT_SESSION_OPTIONS, ...options }; + const unique = new Map(); + for (const asset of assets) if (!unique.has(asset.id)) unique.set(asset.id, asset); + + const dated = [...unique.values()] + .filter(isDated) + .sort((a, b) => a.capturedAt.getTime() - b.capturedAt.getTime() || a.id.localeCompare(b.id)); + const undated = [...unique.values()] + .filter((asset) => !isDated(asset)) + .sort((a, b) => a.id.localeCompare(b.id)); + + const split: DatedAsset[][] = []; + let lastPosition: { lat: number; lon: number } | null = null; + for (const asset of dated) { + const current = split.at(-1); + const previous = current?.at(-1); + let startNew = true; + if (current && previous) { + const gap = asset.capturedAt.getTime() - previous.capturedAt.getTime(); + const jumped = + hasPosition(asset) && + lastPosition !== null && + haversineKm(lastPosition, { lat: asset.gpsLat, lon: asset.gpsLon }) > opts.maxDistanceKm; + const dayChanged = + gap > opts.dayBreakHours * HOUR_MS && + calendarDay(asset.capturedAt, opts.timeZone) !== + calendarDay(previous.capturedAt, opts.timeZone); + startNew = gap > opts.gapMinutes * MINUTE_MS || jumped || dayChanged; + } + if (startNew || !current) { + split.push([asset]); + lastPosition = null; + } else { + current.push(asset); + } + if (hasPosition(asset)) lastPosition = { lat: asset.gpsLat, lon: asset.gpsLon }; + } + + // Merge visually similar neighbours that are close in time. + const merged: DatedAsset[][] = []; + for (const session of split) { + const last = merged.at(-1); + if (last) { + const gap = session[0]!.capturedAt.getTime() - last.at(-1)!.capturedAt.getTime(); + const a = sessionEmbedding(last); + const b = sessionEmbedding(session); + if ( + a && + b && + a.length === b.length && + gap < opts.mergeMaxGapHours * HOUR_MS && + cosineSimilarity(a, b) >= opts.mergeSimilarity + ) { + last.push(...session); + continue; + } + } + merged.push([...session]); + } + + const sessions = merged.map((group) => finishSession(group, `session:${group[0]!.id}`)); + if (undated.length > 0) sessions.push(finishSession(undated, "session:undated")); + return sessions; } export interface VectorItem { @@ -62,26 +204,144 @@ export interface VectorItem { vector: Float32Array; } -/** Agglomerative average-linkage clustering; returns groups with stable keys. */ +/** + * Average-linkage agglomerative clustering cut at a similarity threshold, + * using the nearest-neighbour chain algorithm (O(n²) time and memory). Items + * are sorted by id first and ties resolve to the lower index, so the result + * does not depend on input order. Callers batch very large libraries. + */ +function agglomerate(items: VectorItem[], threshold: number): VectorItem[][] { + const unique = new Map(); + for (const item of items) if (!unique.has(item.id)) unique.set(item.id, item); + const sorted = [...unique.values()].sort((a, b) => a.id.localeCompare(b.id)); + const n = sorted.length; + if (n === 0) return []; + const dim = sorted[0]!.vector.length; + if (sorted.some((item) => item.vector.length !== dim)) throw new Error("vector length mismatch"); + + const sim = new Float64Array(n * n); + for (let i = 0; i < n; i++) { + for (let j = i + 1; j < n; j++) { + const value = cosineSimilarity(sorted[i]!.vector, sorted[j]!.vector); + sim[i * n + j] = value; + sim[j * n + i] = value; + } + } + const size = new Array(n).fill(1); + const members: number[][] = sorted.map((_, i) => [i]); + const active = new Set(members.map((_, i) => i)); + const finished: number[][] = []; + const chain: number[] = []; + + while (active.size > 0) { + if (chain.length === 0) chain.push(Math.min(...active)); + const top = chain.at(-1)!; + const below = chain.length >= 2 ? chain.at(-2)! : -1; + let best = -1; + let bestSim = Number.NEGATIVE_INFINITY; + for (const j of active) { + if (j === top) continue; + const s = sim[top * n + j]!; + if (s > bestSim || (s === bestSim && j < best)) { + best = j; + bestSim = s; + } + } + // Prefer the chain predecessor on ties so the chain cannot cycle. + if (below >= 0 && sim[top * n + below]! === bestSim) best = below; + if (best < 0 || bestSim < threshold) { + // Average linkage is reducible: merges never raise similarity to this cluster again. + active.delete(top); + finished.push(members[top]!); + chain.length = 0; + continue; + } + if (best !== below) { + chain.push(best); + continue; + } + chain.length -= 2; + const a = Math.min(top, below); + const b = Math.max(top, below); + for (const k of active) { + if (k === a || k === b) continue; + const value = + (size[a]! * sim[a * n + k]! + size[b]! * sim[b * n + k]!) / (size[a]! + size[b]!); + sim[a * n + k] = value; + sim[k * n + a] = value; + } + size[a] = size[a]! + size[b]!; + members[a] = [...members[a]!, ...members[b]!]; + active.delete(b); + } + return finished.map((group) => group.sort((x, y) => x - y).map((i) => sorted[i]!)); +} + +function sortGroups(groups: T[]): T[] { + return groups.sort((a, b) => b.ids.length - a.ids.length || a.key.localeCompare(b.key)); +} + +/** + * Agglomerative average-linkage clustering; returns groups with stable keys + * (from the smallest member id), largest first. Default similarity 0.8. + */ export function clusterThemes( - _items: VectorItem[], - _options?: { similarity?: number; minSize?: number }, + items: VectorItem[], + options?: { similarity?: number; minSize?: number }, ): Array<{ key: string; ids: string[] }> { - return notImplemented("clusterThemes"); + const minSize = options?.minSize ?? 1; + return sortGroups( + agglomerate(items, options?.similarity ?? 0.8) + .filter((group) => group.length >= minSize) + .map((group) => ({ key: `theme:${group[0]!.id}`, ids: group.map((item) => item.id) })), + ); } -/** Group face tracks into person suggestions by identity embedding. Local only. */ +/** Group face tracks into person suggestions by identity embedding. Local only. Default similarity 0.6. */ export function clusterFaces( - _tracks: VectorItem[], - _options?: { similarity?: number }, + tracks: VectorItem[], + options?: { similarity?: number }, ): Array<{ key: string; ids: string[]; centroid: Float32Array }> { - return notImplemented("clusterFaces"); + return sortGroups( + agglomerate(tracks, options?.similarity ?? 0.6).map((group) => ({ + key: `person:${group[0]!.id}`, + ids: group.map((item) => item.id), + centroid: meanVector(group.map((item) => item.vector))!, + })), + ); } -/** Rule-based title from date range, place centroid, and dominant album. No model involved. */ +/** + * Rule-based title from the date range and dominant album. Places are not + * named because only rounded coordinates are stored. Returns "" when there is + * nothing to say, so the caller shows its own localized placeholder. + */ export function ruleLabel( - _cluster: { startAt: Date | null; endAt: Date | null; albumNames: string[] }, - _locale?: string, + cluster: { startAt: Date | null; endAt: Date | null; albumNames: string[] }, + locale?: string, + timeZone = "UTC", ): string { - return notImplemented("ruleLabel"); + const counts = new Map(); + for (const raw of cluster.albumNames) { + const name = raw.trim(); + if (name) counts.set(name, (counts.get(name) ?? 0) + 1); + } + const album = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]; + + let dates = ""; + const start = cluster.startAt ?? cluster.endAt; + const end = cluster.endAt ?? cluster.startAt; + if (start && end) { + let formatter: Intl.DateTimeFormat; + try { + formatter = new Intl.DateTimeFormat(locale, { dateStyle: "medium", timeZone }); + } catch { + formatter = new Intl.DateTimeFormat("en", { dateStyle: "medium", timeZone }); + } + dates = + calendarDay(start, timeZone) === calendarDay(end, timeZone) + ? formatter.format(start) + : formatter.formatRange(start, end); + } + return [album?.[0], dates].filter(Boolean).join(" · "); } diff --git a/packages/media/src/compile.ts b/packages/media/src/compile.ts index 913e831..ab16f88 100644 --- a/packages/media/src/compile.ts +++ b/packages/media/src/compile.ts @@ -1,23 +1,43 @@ import type { BrandKit, + CaptionStylePreset, Edl, + EdlAudioClip, + EdlOverlay, + EdlVideoClip, MediaRenderQuality, + MotionTemplateRef, NormalizedBox, PlatformProfile, ReframeKeyframe, RenderPlan, + RenderStep, } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { EdlSchema } from "@rakazo/contracts"; +import { + BLENDED_TRANSITIONS, + clipLengthMs, + edlDurationMs, + introMs, + mainProgramMs, + sortedVideoClips, +} from "./edl.js"; export interface CompileSource { /** Redacted intermediate when redactions apply, otherwise original or proxy. */ path: string; + /** + * Coded size as stored. Rotation 90/270 swaps it for display, because + * ffmpeg auto-rotates while decoding. + */ width: number; height: number; durationMs: number; fps: number; hasAudio: boolean; rotation: 0 | 90 | 180 | 270; + /** A still image (photo). Also assumed when durationMs is 0. */ + still?: boolean; } export interface CaptionWord { @@ -38,28 +58,358 @@ export interface CompileInput { transcripts: Map; /** Saliency or subject points per asset for "auto" reframe. */ subjects: Map>; - /** Pre-rendered motion clips keyed by overlay id, "intro", "outro". */ + /** + * Pre-rendered motion clips keyed by overlay id, "intro", "outro". Missing + * ones become `motion` steps that the renderer runs first. + */ motionClips: Map; files: { lutPath: string | null; watermarkPath: string | null; + /** Brand-kit font artifact id (or family name) → font file. */ fonts: Map; /** artifactId → absolute path for audio clips. */ audio: Map; + /** First brand-kit logo, for logo overlays without a motion clip. */ + logoPath?: string | null; }; /** ffmpeg video encoder name, e.g. libx264, h264_vaapi, h264_videotoolbox. */ videoEncoder: string; + /** Output time of the poster frame. Defaults to a third into the main program. */ + posterAtMs?: number; +} + +// --------------------------------------------------------------------------- +// Escaping +// --------------------------------------------------------------------------- + +/* + * A filtergraph value passes three parsers: + * 1. the graph parser splits filters on `[ ] , ;` and honours `\` and `'`, + * 2. the option parser splits options on `:` (and `=`), again with `\` and `'`, + * 3. drawtext expands `%{...}` and treats `\x` as a literal `x`. + * Each level is escaped with a backslash, innermost first, so any user text + * arrives verbatim and can never open a new option, filter, or expansion. + */ +const escapeDrawtextLevel = (value: string) => value.replace(/[\\%]/g, "\\$&"); +const escapeOptionLevel = (value: string) => value.replace(/[\\':=]/g, "\\$&"); +const escapeGraphLevel = (value: string) => value.replace(/[\\'[\],;]/g, "\\$&"); + +/** Escape text for ffmpeg drawtext/filtergraph values. */ +export function escapeFilterText(text: string): string { + const normalized = text + .replace(/\r\n?/g, "\n") + // biome-ignore lint/suspicious/noControlCharactersInRegex: strip control characters on purpose + .replace(/[\u0000-\u0009\u000b-\u001f\u007f]/g, "") + .trim(); + return escapeGraphLevel(escapeOptionLevel(escapeDrawtextLevel(normalized))); +} + +/** Escape a file path or expression used as a filter option value. */ +export function escapeFilterValue(value: string): string { + if (value.includes("\0")) throw new Error("filter value contains NUL"); + return escapeGraphLevel(escapeOptionLevel(value)); +} + +// --------------------------------------------------------------------------- +// Small helpers +// --------------------------------------------------------------------------- + +/** Fixed-point number without exponent notation. */ +function num(value: number, digits = 3): string { + const fixed = value.toFixed(digits); + return fixed.includes(".") ? fixed.replace(/\.?0+$/, "") || "0" : fixed; +} + +const sec = (ms: number) => num(ms / 1000); + +function joinPath(dir: string, name: string): string { + return `${dir.replace(/[\\/]+$/, "")}/${name}`; +} + +function dirName(file: string): string { + const index = Math.max(file.lastIndexOf("/"), file.lastIndexOf("\\")); + return index > 0 ? file.slice(0, index) : "."; +} + +function brandColor(hex: string | undefined, fallback: string): string { + return hex ? `0x${hex.slice(1).toUpperCase()}` : fallback; +} + +function even(value: number): number { + return Math.max(2, Math.floor(value / 2) * 2); +} + +const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value)); + +function enableBetween(startMs: number, endMs: number): string { + return escapeFilterValue(`between(t,${sec(startMs)},${sec(endMs)})`); +} + +/** atempo accepts 0.5..2 per instance for clean results; chain for the rest. */ +export function atempoChain(speed: number): string[] { + const factors: number[] = []; + let rest = speed; + while (rest > 2) { + factors.push(2); + rest /= 2; + } + while (rest < 0.5) { + factors.push(0.5); + rest /= 0.5; + } + if (Math.abs(rest - 1) > 1e-9 || factors.length === 0) factors.push(rest); + return factors.filter((f) => Math.abs(f - 1) > 1e-9).map((f) => `atempo=${num(f, 6)}`); +} + +// --------------------------------------------------------------------------- +// Reframing +// --------------------------------------------------------------------------- + +interface FrameGeometry { + sourceWidth: number; + sourceHeight: number; + outWidth: number; + outHeight: number; + durationMs: number; +} + +/** Largest crop with the output aspect inside the source, shrunk by scale. */ +function cropSize(geometry: FrameGeometry, scale: number): { w: number; h: number } { + const outAspect = geometry.outWidth / geometry.outHeight; + let w: number; + let h: number; + if (geometry.sourceWidth / geometry.sourceHeight > outAspect) { + h = geometry.sourceHeight; + w = h * outAspect; + } else { + w = geometry.sourceWidth; + h = w / outAspect; + } + const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2); + return { + w: Math.min(geometry.sourceWidth, roundEven(w / scale)), + h: Math.min(geometry.sourceHeight, roundEven(h / scale)), + }; +} + +const AUTO_STEP_MS = 500; +const AUTO_WINDOW_MS = 750; +const AUTO_SMOOTHING = 0.35; +const AUTO_TOLERANCE = 0.01; +const MAX_KEYFRAMES = 64; + +function smooth(values: number[]): number[] { + const forward = [...values]; + for (let i = 1; i < forward.length; i++) { + forward[i] = forward[i - 1]! + AUTO_SMOOTHING * (forward[i]! - forward[i - 1]!); + } + const backward = [...forward]; + for (let i = backward.length - 2; i >= 0; i--) { + backward[i] = backward[i + 1]! + AUTO_SMOOTHING * (backward[i]! - backward[i + 1]!); + } + return backward; +} + +/** Keep only keyframes that linear interpolation cannot reproduce, at most MAX_KEYFRAMES. */ +function simplify(keyframes: ReframeKeyframe[]): ReframeKeyframe[] { + if (keyframes.length <= 2) return keyframes; + const kept: ReframeKeyframe[] = [keyframes[0]!]; + for (let i = 1; i < keyframes.length - 1; i++) { + const last = kept.at(-1)!; + const next = keyframes[i + 1]!; + const point = keyframes[i]!; + const f = (point.tMs - last.tMs) / (next.tMs - last.tMs); + const cx = last.cx + (next.cx - last.cx) * f; + const cy = last.cy + (next.cy - last.cy) * f; + if (Math.abs(cx - point.cx) > AUTO_TOLERANCE || Math.abs(cy - point.cy) > AUTO_TOLERANCE) { + kept.push(point); + } + } + kept.push(keyframes.at(-1)!); + if (kept.length <= MAX_KEYFRAMES) return kept; + const step = (kept.length - 1) / (MAX_KEYFRAMES - 1); + return Array.from({ length: MAX_KEYFRAMES }, (_, i) => kept[Math.round(i * step)]!); } /** - * Pure compiler from a validated EDL to ffmpeg argv steps plus caption files. - * No string ever reaches a shell; text is escaped for filtergraph and ASS, - * every label comes from SafeId values. + * Smoothed crop path that keeps the subject in frame for the output aspect. + * Subjects are weighted by strength in a ±750 ms window every 500 ms, the + * path is smoothed forwards and backwards (no lag), clamped so the crop stays + * inside the source, and reduced to the keyframes that matter. Times are the + * subjects' time base, from 0 to durationMs. */ -export function compileRenderPlan(_input: CompileInput): RenderPlan { - return notImplemented("compileRenderPlan"); +export function autoReframe( + subjects: Array<{ tMs: number; cx: number; cy: number; strength: number }>, + input: FrameGeometry, +): ReframeKeyframe[] { + const crop = cropSize(input, 1); + const halfW = crop.w / input.sourceWidth / 2; + const halfH = crop.h / input.sourceHeight / 2; + const clampX = (x: number) => clamp(x, halfW, 1 - halfW); + const clampY = (y: number) => clamp(y, halfH, 1 - halfH); + const usable = subjects + .filter((s) => s.strength > 0 && Number.isFinite(s.cx) && Number.isFinite(s.cy)) + .sort((a, b) => a.tMs - b.tMs); + if (usable.length === 0) return [{ tMs: 0, cx: clampX(0.5), cy: clampY(0.5), scale: 1 }]; + + const times: number[] = []; + for (let t = 0; t < input.durationMs; t += AUTO_STEP_MS) times.push(t); + times.push(Math.max(0, input.durationMs)); + const xs: number[] = []; + const ys: number[] = []; + for (const t of times) { + let weight = 0; + let x = 0; + let y = 0; + for (const subject of usable) { + const distance = Math.abs(subject.tMs - t); + if (distance > AUTO_WINDOW_MS) continue; + const w = subject.strength * (1 - distance / (AUTO_WINDOW_MS * 2)); + weight += w; + x += subject.cx * w; + y += subject.cy * w; + } + if (weight > 0) { + xs.push(x / weight); + ys.push(y / weight); + } else if (xs.length > 0) { + xs.push(xs.at(-1)!); + ys.push(ys.at(-1)!); + } else { + const nearest = usable.reduce((best, s) => + Math.abs(s.tMs - t) < Math.abs(best.tMs - t) ? s : best, + ); + xs.push(nearest.cx); + ys.push(nearest.cy); + } + } + const sx = smooth(xs); + const sy = smooth(ys); + const round = (v: number) => Math.round(v * 10_000) / 10_000; + return simplify( + times.map((tMs, i) => ({ + tMs: Math.round(tMs), + cx: round(clampX(sx[i]!)), + cy: round(clampY(sy[i]!)), + scale: 1, + })), + ); } +const REFRAME_SCALES = [1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4]; +const FACE_MARGIN = 0.02; + +/** + * Crop path that keeps every given face box outside the output frame, or + * null if impossible. The window is static for the whole shot (a moving + * window could sweep across a face between keyframes): it avoids the union of + * all face boxes plus a 2 % margin, prefers the widest crop, and then the + * position closest to the frame centre. + */ +export function reframeExcluding( + faces: Array<{ tMs: number; box: NormalizedBox }>, + input: FrameGeometry, +): ReframeKeyframe[] | null { + const tMs = faces.reduce((min, face) => Math.min(min, face.tMs), Number.POSITIVE_INFINITY); + const boxes = faces.map(({ box }) => ({ + x0: clamp(box.x - FACE_MARGIN, 0, 1), + y0: clamp(box.y - FACE_MARGIN, 0, 1), + x1: clamp(box.x + box.w + FACE_MARGIN, 0, 1), + y1: clamp(box.y + box.h + FACE_MARGIN, 0, 1), + })); + for (const scale of REFRAME_SCALES) { + const crop = cropSize(input, scale); + const w = crop.w / input.sourceWidth; + const h = crop.h / input.sourceHeight; + const xs = new Set([0.5, w / 2, 1 - w / 2]); + const ys = new Set([0.5, h / 2, 1 - h / 2]); + for (const b of boxes) { + xs.add(b.x0 - w / 2); + xs.add(b.x1 + w / 2); + ys.add(b.y0 - h / 2); + ys.add(b.y1 + h / 2); + } + const candidates: Array<{ cx: number; cy: number; distance: number }> = []; + for (const rawX of xs) { + for (const rawY of ys) { + const cx = clamp(rawX, w / 2, 1 - w / 2); + const cy = clamp(rawY, h / 2, 1 - h / 2); + const clear = boxes.every( + (b) => + cx + w / 2 <= b.x0 + 1e-9 || + cx - w / 2 >= b.x1 - 1e-9 || + cy + h / 2 <= b.y0 + 1e-9 || + cy - h / 2 >= b.y1 - 1e-9, + ); + if (clear) candidates.push({ cx, cy, distance: Math.hypot(cx - 0.5, cy - 0.5) }); + } + } + candidates.sort((a, b) => a.distance - b.distance || a.cx - b.cx || a.cy - b.cy); + const best = candidates[0]; + if (best) { + const round = (v: number) => Math.round(v * 10_000) / 10_000; + return [ + { + tMs: Number.isFinite(tMs) ? Math.round(tMs) : 0, + cx: round(best.cx), + cy: round(best.cy), + scale, + }, + ]; + } + } + return null; +} + +/** Piecewise-linear expression of `t` (seconds) through the given points. */ +function piecewise(points: Array<{ t: number; v: number }>): string { + if (points.length === 1) return num(points[0]!.v, 2); + let expression = num(points.at(-1)!.v, 2); + for (let i = points.length - 2; i >= 0; i--) { + const a = points[i]!; + const b = points[i + 1]!; + const span = Math.max(b.t - a.t, 0.001); + const segment = `${num(a.v, 2)}+(${num(b.v - a.v, 2)})*(t-${num(a.t)})/${num(span)}`; + expression = `if(lt(t,${num(b.t)}),${segment},${expression})`; + } + return `if(lt(t,${num(points[0]!.t)}),${num(points[0]!.v, 2)},${expression})`; +} + +/** crop+scale filters for keyframes in source time; uses the tightest scale of the path. */ +function cropFilters( + keyframes: ReframeKeyframe[], + clip: EdlVideoClip, + geometry: FrameGeometry, +): string[] { + const scale = Math.max(...keyframes.map((k) => k.scale)); + const crop = cropSize(geometry, scale); + const points = [...keyframes] + .sort((a, b) => a.tMs - b.tMs) + .map((k) => ({ + t: Math.max(0, (k.tMs - clip.inMs) / 1000), + x: clamp(k.cx * geometry.sourceWidth - crop.w / 2, 0, geometry.sourceWidth - crop.w), + y: clamp(k.cy * geometry.sourceHeight - crop.h / 2, 0, geometry.sourceHeight - crop.h), + })); + const x = piecewise(points.map((p) => ({ t: p.t, v: p.x }))); + const y = piecewise(points.map((p) => ({ t: p.t, v: p.y }))); + return [ + `crop=w=${crop.w}:h=${crop.h}:x=${escapeFilterValue(x)}:y=${escapeFilterValue(y)}`, + `scale=${geometry.outWidth}:${geometry.outHeight}`, + ]; +} + +function fitFilters(width: number, height: number): string[] { + return [ + `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`, + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, + ]; +} + +// --------------------------------------------------------------------------- +// Captions +// --------------------------------------------------------------------------- + export interface CaptionCue { startMs: number; endMs: number; @@ -68,50 +418,952 @@ export interface CaptionCue { words: CaptionWord[]; } +const CUE_PAUSE_MS = 1000; +const CUE_MAX_MS = 7000; +const CUE_MIN_MS = 500; + +function cleanWord(word: CaptionWord): CaptionWord | null { + const text = word.text.replace(/\s+/g, " ").trim(); + if (!text || !(word.endMs >= word.startMs)) return null; + return { startMs: word.startMs, endMs: word.endMs, text }; +} + +/** Greedy line wrap; a single word longer than the limit gets its own line. */ +function wrapWords(words: CaptionWord[], maxChars: number): CaptionWord[][] { + const lines: CaptionWord[][] = []; + let line: CaptionWord[] = []; + let length = 0; + for (const word of words) { + const extra = line.length === 0 ? word.text.length : length + 1 + word.text.length; + if (line.length > 0 && extra > maxChars) { + lines.push(line); + line = [word]; + length = word.text.length; + } else { + line.push(word); + length = extra; + } + } + if (line.length > 0) lines.push(line); + return lines; +} + +/** + * Group words into cues of at most maxLines lines of maxCharsPerLine. A cue + * also ends at a pause over 1 s, after sentence punctuation, or at 7 s. Each + * cue is shown at least 500 ms unless the next one starts earlier. + */ export function buildCaptionCues( - _words: CaptionWord[], - _options: { maxCharsPerLine: number; maxLines: number }, + words: CaptionWord[], + options: { maxCharsPerLine: number; maxLines: number }, ): CaptionCue[] { - return notImplemented("buildCaptionCues"); + const clean = words + .map(cleanWord) + .filter((w): w is CaptionWord => w !== null) + .sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs); + const groups: CaptionWord[][] = []; + let current: CaptionWord[] = []; + for (const word of clean) { + const last = current.at(-1); + const first = current[0]; + const breaks = + last !== undefined && + first !== undefined && + (word.startMs - last.endMs > CUE_PAUSE_MS || + /[.!?…]["»”)]?$/.test(last.text) || + word.endMs - first.startMs > CUE_MAX_MS || + wrapWords([...current, word], options.maxCharsPerLine).length > options.maxLines); + if (breaks) { + groups.push(current); + current = []; + } + current.push(word); + } + if (current.length > 0) groups.push(current); + + return groups.map((group, index) => { + const startMs = group[0]!.startMs; + const next = groups[index + 1]?.[0]?.startMs ?? Number.POSITIVE_INFINITY; + const spoken = Math.max(...group.map((w) => w.endMs)); + const endMs = Math.max(spoken, Math.min(startMs + CUE_MIN_MS, next)); + return { + startMs, + endMs: Math.max(endMs, startMs + 1), + text: wrapWords(group, options.maxCharsPerLine) + .map((line) => line.map((w) => w.text).join(" ")) + .join("\n"), + words: group, + }; + }); } -export function toSrt(_cues: CaptionCue[]): string { - return notImplemented("toSrt"); +function clock(ms: number, separator: "," | "."): string { + const total = Math.max(0, Math.round(ms)); + const hours = Math.floor(total / 3_600_000); + const minutes = Math.floor((total % 3_600_000) / 60_000); + const seconds = Math.floor((total % 60_000) / 1000); + const millis = total % 1000; + const pad = (v: number, n = 2) => String(v).padStart(n, "0"); + return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}${separator}${pad(millis, 3)}`; } -export function toVtt(_cues: CaptionCue[]): string { - return notImplemented("toVtt"); +/** Cue text without blank lines, which would end a cue early in both formats. */ +function cueLines(text: string): string { + return text + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .join("\n"); } -/** Smoothed crop path that keeps the subject in frame for the output aspect. */ -export function autoReframe( - _subjects: Array<{ tMs: number; cx: number; cy: number; strength: number }>, - _input: { - sourceWidth: number; - sourceHeight: number; - outWidth: number; - outHeight: number; - durationMs: number; +export function toSrt(cues: CaptionCue[]): string { + return cues + .map( + (cue, index) => + `${index + 1}\n${clock(cue.startMs, ",")} --> ${clock(cue.endMs, ",")}\n${cueLines( + cue.text, + ).replace(/-->/g, "->")}\n`, + ) + .join("\n"); +} + +export function toVtt(cues: CaptionCue[]): string { + const escapeCue = (text: string) => + text.replace(/&/g, "&").replace(//g, ">"); + return `WEBVTT\n\n${cues + .map( + (cue) => + `${clock(cue.startMs, ".")} --> ${clock(cue.endMs, ".")}\n${escapeCue(cueLines(cue.text))}\n`, + ) + .join("\n")}`; +} + +/** ASS colour &HAABBGGRR from #RRGGBB. */ +function assColor(hex: string, alpha = 0): string { + const r = hex.slice(1, 3); + const g = hex.slice(3, 5); + const b = hex.slice(5, 7); + return `&H${alpha.toString(16).padStart(2, "0")}${b}${g}${r}`.toUpperCase(); +} + +function assTime(ms: number): string { + const total = Math.max(0, Math.round(ms / 10)); + const hours = Math.floor(total / 360_000); + const minutes = Math.floor((total % 360_000) / 6000); + const seconds = Math.floor((total % 6000) / 100); + const centis = total % 100; + const pad = (v: number) => String(v).padStart(2, "0"); + return `${hours}:${pad(minutes)}:${pad(seconds)}.${pad(centis)}`; +} + +/** Plain text for an ASS event: no override blocks, no escape sequences. */ +function assText(text: string): string { + return text.replace(/\\/g, "\\⁠").replace(/\{/g, "｛").replace(/\}/g, "｝"); +} + +export function toAss( + cues: CaptionCue[], + options: { + width: number; + height: number; + profile: PlatformProfile; + preset: CaptionStylePreset | null; + fallbackTextColor: string; + position: "lower" | "middle" | "upper"; + maxCharsPerLine: number; + wordHighlight: boolean; + language?: string; }, -): ReframeKeyframe[] { - return notImplemented("autoReframe"); +): string { + const { width, height, profile, preset } = options; + const zones = profile.safeZones; + const margin = Math.round(height * 0.02); + const alignment = { lower: 2, middle: 5, upper: 8 }[options.position]; + const marginV = + options.position === "upper" + ? Math.round((zones.topPct / 100) * height) + margin + : options.position === "lower" + ? Math.round((zones.bottomPct / 100) * height) + margin + : 0; + const marginL = Math.round((zones.leftPct / 100) * width) + margin; + const marginR = Math.round((zones.rightPct / 100) * width) + margin; + const textColor = preset?.textColor ?? options.fallbackTextColor; + const highlight = preset?.highlightColor ?? textColor; + const karaoke = options.wordHighlight || preset?.template === "karaoke"; + const bold = preset?.template === "bold_social"; + const fontName = (preset?.fontFamily ?? "Sans").replace(/[,\r\n]/g, " "); + const fontSize = Math.round(((preset?.sizePct ?? 5) / 100) * height); + const background = preset?.backgroundColor; + const style = [ + "Default", + fontName, + fontSize, + assColor(karaoke ? highlight : textColor), + assColor(textColor), + background ? assColor(background) : "&H00000000", + background ? assColor(background, 0x40) : "&H80000000", + bold ? -1 : 0, + 0, + 0, + 0, + 100, + 100, + 0, + 0, + background ? 3 : 1, + bold ? 4 : 2, + 0, + alignment, + marginL, + marginR, + marginV, + 1, + ].join(","); + const transform = (text: string) => + preset?.uppercase ? text.toLocaleUpperCase(options.language) : text; + const events = cues.map((cue) => { + let body: string; + if (karaoke && cue.words.length > 0) { + let cursor = cue.startMs; + body = wrapWords(cue.words, options.maxCharsPerLine) + .map((line) => + line + .map((word) => { + const gap = Math.max(0, Math.round((word.startMs - cursor) / 10)); + const length = Math.max(1, Math.round((word.endMs - word.startMs) / 10)); + cursor = Math.max(cursor, word.endMs); + return `${gap > 0 ? `{\\k${gap}}` : ""}{\\k${length}}${assText(transform(word.text))}`; + }) + .join(" "), + ) + .join("\\N"); + } else { + body = cueLines(transform(cue.text)).split("\n").map(assText).join("\\N"); + } + return `Dialogue: 0,${assTime(cue.startMs)},${assTime(cue.endMs)},Default,,0,0,0,,${body}`; + }); + return [ + "[Script Info]", + "ScriptType: v4.00+", + `PlayResX: ${width}`, + `PlayResY: ${height}`, + "WrapStyle: 2", + "ScaledBorderAndShadow: yes", + "", + "[V4+ Styles]", + "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding", + `Style: ${style}`, + "", + "[Events]", + "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text", + ...events, + "", + ].join("\n"); } -/** Crop path that keeps every given face box outside the output frame, or null if impossible. */ -export function reframeExcluding( - _faces: Array<{ tMs: number; box: NormalizedBox }>, - _input: { - sourceWidth: number; - sourceHeight: number; - outWidth: number; - outHeight: number; - durationMs: number; - }, -): ReframeKeyframe[] | null { - return notImplemented("reframeExcluding"); +/** Caption words on the output timeline, from each unmuted clip's transcript. */ +export function outputCaptionWords( + edl: Edl, + transcripts: Map, +): CaptionWord[] { + const offset = introMs(edl); + const words: CaptionWord[] = []; + for (const clip of sortedVideoClips(edl)) { + if (clip.muted) continue; + for (const word of transcripts.get(clip.assetId) ?? []) { + const start = Math.max(word.startMs, clip.inMs); + const end = Math.min(word.endMs, clip.outMs); + if (word.startMs < clip.inMs || start >= clip.outMs || end < start) continue; + words.push({ + startMs: Math.round(offset + clip.atMs + (start - clip.inMs) / clip.speed), + endMs: Math.round(offset + clip.atMs + (end - clip.inMs) / clip.speed), + text: word.text, + }); + } + } + return words; } -/** Escape text for ffmpeg drawtext/filtergraph values. */ -export function escapeFilterText(_text: string): string { - return notImplemented("escapeFilterText"); +// --------------------------------------------------------------------------- +// Compiler +// --------------------------------------------------------------------------- + +const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo"; +const XFADE_KIND = { crossfade: "fade", dip: "fadeblack", whip: "slideleft" } as const; +const MAX_INLINE_GRAPH = 60_000; + +function encoderFamily(encoder: string): "h264" | "hevc" | null { + if (/^(libx264|h264_)/.test(encoder)) return "h264"; + if (/^(libx265|hevc_)/.test(encoder)) return "hevc"; + return null; +} + +function encodingArgs(input: CompileInput): string[] { + const { profile, quality, videoEncoder } = input; + const family = encoderFamily(videoEncoder); + if (family !== profile.video.codec) { + throw new Error(`encoder ${videoEncoder} does not produce ${profile.video.codec}`); + } + const draft = quality === "draft"; + const maxKbps = draft + ? Math.min(profile.video.maxBitrateKbps, 3000) + : profile.video.maxBitrateKbps; + const args = ["-c:v", videoEncoder]; + if (videoEncoder === "libx264" || videoEncoder === "libx265") { + args.push("-preset", draft ? "veryfast" : "medium", "-crf", draft ? "28" : "20"); + if (videoEncoder === "libx264") args.push("-profile:v", "high"); + } else { + args.push("-b:v", `${Math.round(maxKbps * 0.8)}k`); + } + args.push("-maxrate", `${maxKbps}k`, "-bufsize", `${maxKbps * 2}k`); + if (!videoEncoder.endsWith("_vaapi")) args.push("-pix_fmt", "yuv420p"); + args.push( + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + ); + // Drop SEI messages, which carry encoder name, version and settings. + args.push( + "-bsf:v", + family === "h264" ? "filter_units=remove_types=6" : "filter_units=remove_types=39|40", + ); + if (family === "hevc") args.push("-tag:v", "hvc1"); + args.push("-r", num(input.edl.output.fps, 3)); + args.push("-c:a", "aac", "-b:a", draft ? "128k" : "192k", "-ar", "48000", "-ac", "2"); + return args; +} + +/** Strip every inherited tag, chapter, and encoder string; only the plan's title is written. */ +function metadataArgs(title: string | undefined): string[] { + const args = [ + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + ]; + const clean = title?.replace(/[\0\r\n]+/g, " ").trim(); + if (clean) args.push("-metadata", `title=${clean}`); + return args; +} + +class Inputs { + readonly args: string[] = []; + private count = 0; + add(...args: string[]): number { + if (args.some((arg) => arg.includes("\0"))) throw new Error("argument contains NUL"); + this.args.push(...args); + return this.count++; + } +} + +function captionPreset(edl: Edl, kit: BrandKit | null): CaptionStylePreset | null { + const presets = kit?.captionPresets ?? []; + return presets.find((p) => p.id === edl.captions?.presetId) ?? presets[0] ?? null; +} + +function pickFont(kit: BrandKit | null, fonts: Map, role: string): string | null { + const preferred = kit?.fonts.find((font) => font.role === role) ?? kit?.fonts[0]; + if (preferred) { + const path = fonts.get(preferred.artifactId) ?? fonts.get(preferred.family); + if (path) return path; + } + return fonts.values().next().value ?? null; +} + +/** + * Pure compiler from a validated EDL to ffmpeg argv steps plus caption files. + * No string ever reaches a shell; text is escaped for filtergraph and ASS, + * every label comes from SafeId values. + * + * Loudness: a single loudnorm pass with linear=true targets the profile (or + * plan) loudness and true peak. Without measured values loudnorm runs in its + * dynamic mode; a renderer may run a measurement pass first and append the + * measured_* options to that filter. + * + * Face, mark, and region redactions are applied before compilation: sources + * already point at redacted intermediates. + */ +export function compileRenderPlan(input: CompileInput): RenderPlan { + const edl = EdlSchema.parse(input.edl); + const { profile, brandKit: kit, workDir } = input; + const W = edl.output.width; + const H = edl.output.height; + const F = edl.output.fps; + const totalMs = edlDurationMs(edl); + const intro = introMs(edl); + const mainMs = mainProgramMs(edl); + const steps: RenderStep[] = []; + const outputs: RenderPlan["outputs"] = []; + const inputs = new Inputs(); + const graph: string[] = []; + let motionCount = 0; + + const motionInput = (key: string, template: MotionTemplateRef, width: number, height: number) => { + let path = input.motionClips.get(key); + if (!path) { + motionCount += 1; + path = joinPath(workDir, `motion-${motionCount}.mov`); + steps.push({ + kind: "motion", + id: `motion-${motionCount}`, + template, + width, + height, + fps: F, + outPath: path, + }); + } + return inputs.add("-i", path); + }; + + const finishVideo = (lengthMs: number) => [ + `fps=${num(F, 3)}`, + "format=yuv420p", + "tpad=stop_mode=clone:stop_duration=2", + `trim=duration=${sec(lengthMs)}`, + "setpts=PTS-STARTPTS", + ]; + const finishAudio = (lengthMs: number) => [ + AUDIO_FORMAT, + "apad", + `atrim=duration=${sec(lengthMs)}`, + "asetpts=PTS-STARTPTS", + ]; + const silence = (label: string, lengthMs: number) => + graph.push(`anullsrc=r=48000:cl=stereo,${finishAudio(lengthMs).join(",")}[${label}]`); + + // Per-clip chains. + interface Part { + v: string; + a: string; + lengthMs: number; + } + const clipParts = new Map(); + for (const clip of sortedVideoClips(edl)) { + const source = input.sources.get(clip.assetId); + if (!source) throw new Error(`no source for asset in clip ${clip.id}`); + const still = source.still === true || source.durationMs === 0; + const sourceLength = clip.outMs - clip.inMs; + const lengthMs = clipLengthMs(clip); + const index = still + ? inputs.add( + "-loop", + "1", + "-framerate", + num(F, 3), + "-t", + sec(sourceLength), + "-i", + source.path, + ) + : inputs.add("-ss", sec(clip.inMs), "-t", sec(sourceLength), "-i", source.path); + const rotated = source.rotation === 90 || source.rotation === 270; + const geometry: FrameGeometry = { + sourceWidth: rotated ? source.height : source.width, + sourceHeight: rotated ? source.width : source.height, + outWidth: W, + outHeight: H, + durationMs: sourceLength, + }; + const video: string[] = []; + if (clip.stabilize && !still) video.push("deshake"); + if (clip.reframe === "none") { + video.push(...fitFilters(W, H)); + } else { + const keyframes = + clip.reframe === "auto" + ? autoReframe( + (input.subjects.get(clip.assetId) ?? []) + .filter( + (s) => + s.tMs >= clip.inMs - AUTO_WINDOW_MS && s.tMs <= clip.outMs + AUTO_WINDOW_MS, + ) + .map((s) => ({ ...s, tMs: s.tMs - clip.inMs })), + geometry, + ).map((k) => ({ ...k, tMs: k.tMs + clip.inMs })) + : clip.reframe; + video.push(...cropFilters(keyframes, clip, geometry)); + } + video.push("setsar=1", `setpts=(PTS-STARTPTS)/${num(clip.speed, 6)}`); + const color = clip.color; + if (color) { + const brightness = clamp(color.exposure * 0.1, -1, 1); + if (brightness !== 0 || color.contrast !== 1 || color.saturation !== 1) { + video.push( + `eq=brightness=${num(brightness)}:contrast=${num(color.contrast)}:saturation=${num(color.saturation)}`, + ); + } + if (color.temperature !== 0) { + const shift = num(color.temperature * 0.1); + const inverse = num(-color.temperature * 0.1); + video.push(`colorbalance=rm=${shift}:bm=${inverse}`); + } + if (color.useBrandLut) { + if (!input.files.lutPath) throw new Error(`clip ${clip.id} needs the brand LUT file`); + video.push(`lut3d=file=${escapeFilterValue(input.files.lutPath)}`); + } + } + video.push(...finishVideo(lengthMs)); + graph.push(`[${index}:v]${video.join(",")}[v_${clip.id}]`); + + if (source.hasAudio && !clip.muted && !still) { + const audio = ["asetpts=PTS-STARTPTS", ...atempoChain(clip.speed), "aresample=48000"]; + graph.push(`[${index}:a]${[...audio, ...finishAudio(lengthMs)].join(",")}[a_${clip.id}]`); + } else { + silence(`a_${clip.id}`, lengthMs); + } + clipParts.set(clip.id, { v: `v_${clip.id}`, a: `a_${clip.id}`, lengthMs }); + } + + // Main program: concatenate runs, blend where transitions ask for it, fill gaps with black. + const blends = new Map( + edl.transitions + .filter((t) => BLENDED_TRANSITIONS.has(t.kind) && t.durationMs > 0) + .map((t) => [`${t.fromClipId}>${t.toClipId}`, t] as const), + ); + let mixCount = 0; + const nextLabel = (prefix: string) => `${prefix}${++mixCount}`; + let accumulated: Part | null = null; + let queue: Part[] = []; + const flush = () => { + const parts = accumulated ? [accumulated, ...queue] : queue; + queue = []; + if (parts.length <= 1) { + accumulated = parts[0] ?? accumulated; + return; + } + const v = nextLabel("cv"); + const a = nextLabel("ca"); + graph.push( + `${parts.map((p) => `[${p.v}][${p.a}]`).join("")}concat=n=${parts.length}:v=1:a=1[${v}][${a}]`, + ); + accumulated = { v, a, lengthMs: parts.reduce((sum, p) => sum + p.lengthMs, 0) }; + }; + let cursor = 0; + let previous: EdlVideoClip | null = null; + for (const clip of sortedVideoClips(edl)) { + const part = clipParts.get(clip.id)!; + const blend = previous ? blends.get(`${previous.id}>${clip.id}`) : undefined; + if (blend) { + flush(); + const base = accumulated as Part | null; + if (!base) throw new Error("transition without a preceding clip"); + const v = nextLabel("xv"); + const a = nextLabel("xa"); + const kind = XFADE_KIND[blend.kind as keyof typeof XFADE_KIND]; + graph.push( + `[${base.v}][${part.v}]xfade=transition=${kind}:duration=${sec(blend.durationMs)}:offset=${sec( + base.lengthMs - blend.durationMs, + )}[${v}]`, + `[${base.a}][${part.a}]acrossfade=d=${sec(blend.durationMs)}:c1=tri:c2=tri[${a}]`, + ); + accumulated = { v, a, lengthMs: base.lengthMs + part.lengthMs - blend.durationMs }; + } else { + const gap = clip.atMs - cursor; + if (gap > 1) { + const v = nextLabel("gv"); + const a = nextLabel("ga"); + graph.push(`color=c=black:s=${W}x${H}:r=${num(F, 3)},${finishVideo(gap).join(",")}[${v}]`); + silence(a, gap); + queue.push({ v, a, lengthMs: gap }); + } + queue.push(part); + } + cursor = Math.max(cursor, clip.atMs + part.lengthMs); + previous = clip; + } + flush(); + const main = accumulated as Part | null; + if (!main) throw new Error("plan has no video clips"); + + let mainAudio = main.a; + const voice = edl.redactions.voice; + if (voice) { + const ratio = 2 ** (voice.semitones / 12); + const label = nextLabel("vd"); + const filter = + voice.method === "pitch" + ? `rubberband=pitch=${num(ratio, 6)}:formant=preserved` + : `asetrate=${num(48000 * ratio, 2)},aresample=48000,atempo=${num(1 / ratio, 6)}`; + graph.push(`[${mainAudio}]${filter},${finishAudio(main.lengthMs).join(",")}[${label}]`); + mainAudio = label; + } + + // Intro and outro. + const program: Part[] = []; + const bookend = (key: "intro" | "outro", ref: MotionTemplateRef | undefined) => { + if (!ref || ref.durationMs <= 0) return; + const index = motionInput(key, ref, W, H); + const v = `${key}_v`; + const a = `${key}_a`; + graph.push( + `[${index}:v]${[...fitFilters(W, H), "setsar=1", ...finishVideo(ref.durationMs)].join(",")}[${v}]`, + ); + silence(a, ref.durationMs); + program.push({ v, a, lengthMs: ref.durationMs }); + }; + bookend("intro", edl.intro); + program.push({ v: main.v, a: mainAudio, lengthMs: main.lengthMs }); + bookend("outro", edl.outro); + let videoLabel = main.v; + let programAudio = mainAudio; + if (program.length > 1) { + videoLabel = "prog_v"; + programAudio = "prog_a"; + graph.push( + `${program.map((p) => `[${p.v}][${p.a}]`).join("")}concat=n=${program.length}:v=1:a=1[prog_v][prog_a]`, + ); + } + + // Overlays on the output timeline. + const zones = profile.safeZones; + const margin = Math.round(W * 0.02); + const safeLeft = Math.round((zones.leftPct / 100) * W) + margin; + const safeRight = Math.round((zones.rightPct / 100) * W) + margin; + const safeTop = Math.round((zones.topPct / 100) * H) + margin; + const safeBottom = Math.round((zones.bottomPct / 100) * H) + margin; + const textColor = brandColor(kit?.colors.text, "white"); + const overlays = [...edl.overlays].sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)); + const imageOverlay = ( + path: string, + overlay: EdlOverlay, + widthPx: number, + opacity: number, + position: { x: string; y: string }, + ) => { + const index = inputs.add("-i", path); + const label = nextLabel("img"); + const out = nextLabel("ov"); + const alpha = opacity < 1 ? `,colorchannelmixer=aa=${num(opacity)}` : ""; + graph.push( + `[${index}:v]format=rgba,scale=${even(widthPx)}:-2${alpha}[${label}]`, + `[${videoLabel}][${label}]overlay=x=${escapeFilterValue(position.x)}:y=${escapeFilterValue( + position.y, + )}:enable=${enableBetween(overlay.atMs, overlay.atMs + overlay.durationMs)}[${out}]`, + ); + videoLabel = out; + }; + const boxPosition = (box: NormalizedBox) => ({ + x: String(Math.round(box.x * W)), + y: String(Math.round(box.y * H)), + }); + const cornerPosition = (corner: string) => ({ + x: corner.endsWith("left") + ? String(safeLeft) + : corner === "center" + ? "(main_w-overlay_w)/2" + : `main_w-overlay_w-${safeRight}`, + y: corner.startsWith("top") + ? String(safeTop) + : corner === "center" + ? "(main_h-overlay_h)/2" + : `main_h-overlay_h-${safeBottom}`, + }); + + for (const overlay of overlays) { + const endMs = overlay.atMs + overlay.durationMs; + const motionPath = input.motionClips.get(overlay.id); + if (overlay.motion || motionPath) { + const box = overlay.box; + const width = box ? even(box.w * W) : W; + const height = box ? even(box.h * H) : H; + const template = overlay.motion ?? { + templateId: "overlay", + params: {}, + durationMs: overlay.durationMs, + }; + const index = motionInput(overlay.id, template, width, height); + const label = nextLabel("mo"); + const out = nextLabel("ov"); + const position = box ? boxPosition(box) : { x: "0", y: "0" }; + graph.push( + `[${index}:v]format=yuva420p,scale=${width}:${height},setpts=PTS-STARTPTS+${sec(overlay.atMs)}/TB[${label}]`, + `[${videoLabel}][${label}]overlay=x=${position.x}:y=${position.y}:eof_action=pass:enable=${enableBetween( + overlay.atMs, + endMs, + )}[${out}]`, + ); + videoLabel = out; + continue; + } + if (overlay.kind === "watermark") { + const watermark = kit?.watermark; + if (!watermark || !input.files.watermarkPath) { + throw new Error(`overlay ${overlay.id} needs the brand watermark file`); + } + const width = overlay.box ? overlay.box.w * W : watermark.scale * W; + imageOverlay( + input.files.watermarkPath, + overlay, + width, + watermark.opacity, + overlay.box ? boxPosition(overlay.box) : cornerPosition(watermark.position), + ); + continue; + } + if (overlay.kind === "logo") { + if (!input.files.logoPath) throw new Error(`overlay ${overlay.id} needs a logo file`); + const width = overlay.box ? overlay.box.w * W : 0.15 * W; + imageOverlay( + input.files.logoPath, + overlay, + width, + 1, + overlay.box ? boxPosition(overlay.box) : cornerPosition("top_right"), + ); + continue; + } + if (overlay.kind === "progress_bar") { + const barHeight = Math.max(4, even(H * 0.006)); + const label = nextLabel("pb"); + const out = nextLabel("ov"); + const y = overlay.box ? Math.round(overlay.box.y * H) : H - safeBottom - barHeight; + const x = `-overlay_w+main_w*min(1,(t-${sec(overlay.atMs)})/${sec(Math.max(1, overlay.durationMs))})`; + graph.push( + `color=c=${brandColor(kit?.colors.accent, "white")}:s=${W}x${barHeight}:r=${num(F, 3)}:d=${sec(totalMs)}[${label}]`, + `[${videoLabel}][${label}]overlay=x=${escapeFilterValue(x)}:y=${y}:enable=${enableBetween( + overlay.atMs, + endMs, + )}[${out}]`, + ); + videoLabel = out; + continue; + } + // text, lower_third, cta_end_card + const text = overlay.text?.trim(); + if (!text) throw new Error(`overlay ${overlay.id} has no text and no motion clip`); + const lines = text.split(/\r?\n/).length; + const fontSize = overlay.box + ? Math.max(8, Math.round((overlay.box.h * H * 0.8) / lines)) + : Math.round(H * (overlay.kind === "cta_end_card" ? 0.05 : 0.04)); + const position = overlay.box + ? boxPosition(overlay.box) + : { + x: "(w-text_w)/2", + y: + overlay.kind === "lower_third" + ? `h-text_h-${safeBottom + Math.round(H * 0.08)}` + : overlay.kind === "cta_end_card" + ? "(h-text_h)/2" + : String(safeTop + Math.round(H * 0.05)), + }; + const font = pickFont( + kit, + input.files.fonts, + overlay.kind === "cta_end_card" ? "heading" : "body", + ); + const options = [ + ...(font ? [`fontfile=${escapeFilterValue(font)}`] : []), + `text=${escapeFilterText(text)}`, + `fontsize=${fontSize}`, + `fontcolor=${textColor}`, + `x=${escapeFilterValue(position.x)}`, + `y=${escapeFilterValue(position.y)}`, + ]; + if (overlay.kind !== "text") { + options.push( + "box=1", + `boxcolor=${brandColor(kit?.colors.primary, "black")}@0.85`, + `boxborderw=${Math.round(fontSize * 0.4)}`, + ); + } + options.push(`enable=${enableBetween(overlay.atMs, endMs)}`); + const out = nextLabel("ov"); + graph.push(`[${videoLabel}]drawtext=${options.join(":")}[${out}]`); + videoLabel = out; + } + + // Captions. + const captions = edl.captions; + if (captions && captions.mode !== "none") { + const cues = buildCaptionCues(outputCaptionWords(edl, input.transcripts), { + maxCharsPerLine: captions.maxCharsPerLine, + maxLines: profile.captions.maxLines, + }); + if (cues.length > 0) { + if (captions.mode === "burned" || captions.mode === "both") { + const preset = captionPreset(edl, kit); + const assPath = joinPath(workDir, "captions.ass"); + steps.push({ + kind: "write_file", + id: "captions-ass", + path: assPath, + content: toAss(cues, { + width: W, + height: H, + profile, + preset, + fallbackTextColor: kit?.colors.text ?? "#FFFFFF", + position: captions.position, + maxCharsPerLine: captions.maxCharsPerLine, + wordHighlight: captions.wordHighlight, + language: captions.language, + }), + }); + const font = pickFont(kit, input.files.fonts, "caption"); + const fontsDir = font ? `:fontsdir=${escapeFilterValue(dirName(font))}` : ""; + const out = nextLabel("cap"); + graph.push(`[${videoLabel}]ass=filename=${escapeFilterValue(assPath)}${fontsDir}[${out}]`); + videoLabel = out; + } + if (captions.mode === "sidecar" || captions.mode === "both") { + const srt = joinPath(workDir, "captions.srt"); + const vtt = joinPath(workDir, "captions.vtt"); + steps.push( + { kind: "write_file", id: "captions-srt", path: srt, content: toSrt(cues) }, + { kind: "write_file", id: "captions-vtt", path: vtt, content: toVtt(cues) }, + ); + outputs.push({ kind: "subtitles_srt", path: srt }, { kind: "subtitles_vtt", path: vtt }); + } + } + } + const finalVideo = input.videoEncoder.endsWith("_vaapi") + ? "format=nv12,hwupload" + : "format=yuv420p"; + graph.push(`[${videoLabel}]${finalVideo}[vout]`); + + // Audio: program (+ voice-over) is speech; clips marked duckUnderSpeech are compressed under it. + const speech: string[] = [programAudio]; + const ducked: string[] = []; + const beds: string[] = []; + const audioClips = [...edl.tracks.audio].sort( + (a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id), + ); + for (const clip of audioClips) { + const path = input.files.audio.get(clip.artifactId); + if (!path) throw new Error(`no file for audio clip ${clip.id}`); + const length = clip.outMs - clip.inMs; + const index = inputs.add("-ss", sec(clip.inMs), "-t", sec(length), "-i", path); + graph.push(`[${index}:a]${audioClipFilters(clip, length).join(",")}[ac_${clip.id}]`); + const label = `ac_${clip.id}`; + if (clip.kind === "voiceover") speech.push(label); + else if (clip.duckUnderSpeech) ducked.push(label); + else beds.push(label); + } + const mix = (labels: string[], out: string) => { + if (labels.length === 1) { + graph.push(`[${labels[0]}]anull[${out}]`); + } else { + graph.push( + `${labels.map((l) => `[${l}]`).join("")}amix=inputs=${labels.length}:duration=first:normalize=0[${out}]`, + ); + } + }; + const finalInputs: string[] = []; + if (ducked.length > 0) { + mix(speech, "speech"); + graph.push("[speech]asplit=2[speech_mix][speech_key]"); + mix(ducked, "duck_bus"); + graph.push( + "[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked]", + ); + finalInputs.push("speech_mix", "ducked"); + } else { + finalInputs.push(...speech); + } + finalInputs.push(...beds); + const targetLufs = edl.targetLufs ?? profile.audio.targetLufs; + const loudnorm = `loudnorm=I=${num(targetLufs, 1)}:TP=${num(profile.audio.truePeakDb, 1)}:LRA=11:linear=true`; + const mixed = finalInputs.length > 1 ? "mixed" : finalInputs[0]!; + if (finalInputs.length > 1) mix(finalInputs, "mixed"); + graph.push(`[${mixed}]${loudnorm},aresample=48000,${AUDIO_FORMAT}[aout]`); + + // Render step. + const graphText = graph.join(";"); + const graphArgs = + graphText.length > MAX_INLINE_GRAPH + ? ["-filter_complex_script", joinPath(workDir, "filtergraph.txt")] + : ["-filter_complex", graphText]; + if (graphText.length > MAX_INLINE_GRAPH) { + steps.push({ + kind: "write_file", + id: "filtergraph", + path: joinPath(workDir, "filtergraph.txt"), + content: graphText, + }); + } + const videoPath = joinPath(workDir, "output.mp4"); + steps.push({ + kind: "ffmpeg", + id: "render", + argv: [ + "-hide_banner", + "-nostdin", + "-y", + ...inputs.args, + ...graphArgs, + "-map", + "[vout]", + "-map", + "[aout]", + ...encodingArgs(input), + ...metadataArgs(edl.metadata.title), + "-t", + sec(totalMs), + "-movflags", + "+faststart", + videoPath, + ], + produces: [videoPath], + durationMs: totalMs, + }); + outputs.unshift({ kind: "video", path: videoPath }); + + if (profile.export.poster) { + const posterPath = joinPath(workDir, "poster.jpg"); + const at = clamp(input.posterAtMs ?? intro + mainMs / 3, 0, Math.max(0, totalMs - 1)); + steps.push({ + kind: "ffmpeg", + id: "poster", + argv: [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + sec(at), + "-i", + videoPath, + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + posterPath, + ], + produces: [posterPath], + durationMs: 0, + }); + outputs.push({ kind: "poster", path: posterPath }); + } + + return { quality: input.quality, workDir, steps, outputs }; +} + +function audioClipFilters(clip: EdlAudioClip, lengthMs: number): string[] { + const filters = ["asetpts=PTS-STARTPTS", "aresample=48000", AUDIO_FORMAT]; + if (clip.gainDb !== 0) filters.push(`volume=${num(clip.gainDb, 2)}dB`); + if (clip.fadeInMs > 0) filters.push(`afade=t=in:st=0:d=${sec(clip.fadeInMs)}`); + if (clip.fadeOutMs > 0) { + filters.push( + `afade=t=out:st=${sec(Math.max(0, lengthMs - clip.fadeOutMs))}:d=${sec(clip.fadeOutMs)}`, + ); + } + if (clip.atMs > 0) filters.push(`adelay=${clip.atMs}|${clip.atMs}`); + return filters; } diff --git a/packages/media/src/edl.ts b/packages/media/src/edl.ts index 4c3a06e..384f3d3 100644 --- a/packages/media/src/edl.ts +++ b/packages/media/src/edl.ts @@ -1,5 +1,30 @@ -import type { BrandKit, Edl, EdlError, MediaAssetKind, PlatformProfile } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import type { + AspectRatio, + BrandKit, + Edl, + EdlError, + EdlTransition, + EdlVideoClip, + MediaAssetKind, + MediaProjectGoal, + NormalizedBox, + PlatformProfile, +} from "@rakazo/contracts"; +import { EdlSchema } from "@rakazo/contracts"; + +/* + * Timeline model shared by validation, compilation, gate and auto-fix: + * + * - The output is intro + main program + outro, played back to back. + * - `tracks.video[].atMs` is a position on the main program, which starts + * after the intro. A clip lasts (outMs - inMs) / speed. + * - Clips may only overlap where a crossfade, dip or whip transition joins + * them, and then by exactly the transition duration. Cuts and match cuts + * never overlap. + * - Overlays and audio clips are positioned on the output timeline (intro + * included), so music can run under the intro. + * - Reframe keyframes are in source time of the clip's asset. + */ export interface EdlContext { profile: PlatformProfile; @@ -18,28 +43,384 @@ export interface EdlContext { templateIds: Set; /** Segments the user reserved for the paid release. */ paidOnly: Array<{ assetId: string; startMs: number; endMs: number }>; + /** Project goal; a teaser never uses paid-only segments, whatever the profile. */ + goal?: MediaProjectGoal; } export type EdlValidation = { ok: true; edl: Edl } | { ok: false; errors: EdlError[] }; +/** Transitions rendered as a timed blend (xfade). Cuts and match cuts are plain concatenation. */ +export const BLENDED_TRANSITIONS: ReadonlySet = new Set([ + "crossfade", + "dip", + "whip", +]); + +export const ASPECT_RATIO_VALUES: Record = { + "9:16": 9 / 16, + "4:5": 4 / 5, + "1:1": 1, + "16:9": 16 / 9, + "4:3": 4 / 3, + "21:9": 21 / 9, +}; + +const ASPECT_TOLERANCE = 0.01; + +/** Whether width × height matches one of the allowed aspect ratios within 1 %. */ +export function matchesAspect(width: number, height: number, allowed: AspectRatio[]): boolean { + const actual = width / height; + return allowed.some( + (ratio) => + Math.abs(actual - ASPECT_RATIO_VALUES[ratio]) / ASPECT_RATIO_VALUES[ratio] <= + ASPECT_TOLERANCE, + ); +} + +/** Output length of one video clip after speed change. */ +export function clipLengthMs(clip: Pick): number { + return Math.max(0, Math.round((clip.outMs - clip.inMs) / clip.speed)); +} + +/** End of the main program (excluding intro and outro). */ +export function mainProgramMs(edl: Edl): number { + let end = 0; + for (const clip of edl.tracks.video) end = Math.max(end, clip.atMs + clipLengthMs(clip)); + return end; +} + +export function introMs(edl: Edl): number { + return edl.intro?.durationMs ?? 0; +} + +/** Output length in ms including intro, outro, speed, and transition overlap. */ +export function edlDurationMs(edl: Edl): number { + return introMs(edl) + mainProgramMs(edl) + (edl.outro?.durationMs ?? 0); +} + +/** Every clip visible at an output timestamp (two during a blended transition), earliest first. */ +export function locateOutputTimeAll( + edl: Edl, + outputMs: number, +): Array<{ clipId: string; assetId: string; sourceMs: number }> { + const t = outputMs - introMs(edl); + if (t < 0) return []; + return [...edl.tracks.video] + .filter((clip) => t >= clip.atMs && t < clip.atMs + clipLengthMs(clip)) + .sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)) + .map((clip) => ({ + clipId: clip.id, + assetId: clip.assetId, + sourceMs: Math.min(clip.outMs, Math.round(clip.inMs + (t - clip.atMs) * clip.speed)), + })); +} + +/** + * Map an output timestamp back to the clip and source time it shows. During a + * blended transition the incoming clip wins; intro and outro map to null. + */ +export function locateOutputTime( + edl: Edl, + outputMs: number, +): { clipId: string; assetId: string; sourceMs: number } | null { + return locateOutputTimeAll(edl, outputMs).at(-1) ?? null; +} + +/** Whether a normalized box lies inside the profile's safe zones. */ +export function boxInsideSafeZone(box: NormalizedBox, profile: PlatformProfile): boolean { + const epsilon = 1e-6; + const zones = profile.safeZones; + return ( + box.x >= zones.leftPct / 100 - epsilon && + box.y >= zones.topPct / 100 - epsilon && + box.x + box.w <= 1 - zones.rightPct / 100 + epsilon && + box.y + box.h <= 1 - zones.bottomPct / 100 + epsilon + ); +} + +function formatPath(path: ReadonlyArray): string { + let out = ""; + for (const part of path) { + if (typeof part === "number") out += `[${part}]`; + else out += out ? `.${String(part)}` : String(part); + } + return out || "(root)"; +} + +/** Video clips in timeline order, ties broken by id for determinism. */ +export function sortedVideoClips(edl: Edl): EdlVideoClip[] { + return [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)); +} + /** * Parse with EdlSchema, then check references, ranges, overlaps, durations, * safe zones, paid-only use (teaser-style profiles), and music licensing. * Every failure is reported; nothing is silently corrected. */ -export function validateEdl(_input: unknown, _context: EdlContext): EdlValidation { - return notImplemented("validateEdl"); -} +export function validateEdl(input: unknown, context: EdlContext): EdlValidation { + const parsed = EdlSchema.safeParse(input); + if (!parsed.success) { + return { + ok: false, + errors: parsed.error.issues.map((issue) => ({ + code: "schema", + path: formatPath(issue.path), + message: issue.message, + })), + }; + } + const edl = parsed.data; + const errors: EdlError[] = []; + const fail = (code: EdlError["code"], path: string, message: string) => + errors.push({ code, path, message }); + const { profile } = context; -/** Output length in ms including intro, outro, speed, and transition overlap. */ -export function edlDurationMs(_edl: Edl): number { - return notImplemented("edlDurationMs"); -} + if (edl.output.profileId !== profile.id) { + fail("profile_mismatch", "output.profileId", `plan targets ${edl.output.profileId}`); + } + if (!matchesAspect(edl.output.width, edl.output.height, profile.video.aspectRatios)) { + fail( + "profile_mismatch", + "output", + `${edl.output.width}x${edl.output.height} is not one of ${profile.video.aspectRatios.join(", ")}`, + ); + } + if (edl.output.fps > profile.video.fps * 1.001) { + fail("profile_mismatch", "output.fps", `fps above profile maximum ${profile.video.fps}`); + } -/** Map an output timestamp back to the clip and source time it shows. */ -export function locateOutputTime( - _edl: Edl, - _outputMs: number, -): { clipId: string; assetId: string; sourceMs: number } | null { - return notImplemented("locateOutputTime"); + const seenIds = new Map(); + const claimId = (id: string, path: string) => { + const previous = seenIds.get(id); + if (previous) fail("schema", path, `id ${id} already used at ${previous}`); + else seenIds.set(id, path); + }; + + const clipsById = new Map(); + edl.tracks.video.forEach((clip, index) => { + const path = `tracks.video[${index}]`; + claimId(clip.id, `${path}.id`); + clipsById.set(clip.id, clip); + if (clip.outMs <= clip.inMs) fail("range_empty", path, "outMs must be after inMs"); + const asset = context.assets.get(clip.assetId); + if (!asset) { + fail("unknown_asset", `${path}.assetId`, `asset ${clip.assetId} is not available`); + } else if (asset.durationMs !== null && asset.kind !== "photo") { + if (clip.outMs > asset.durationMs) { + fail( + "range_out_of_asset", + `${path}.outMs`, + `outMs ${clip.outMs} exceeds asset duration ${asset.durationMs}`, + ); + } + if (clip.inMs >= asset.durationMs) { + fail("range_out_of_asset", `${path}.inMs`, `inMs ${clip.inMs} is past the asset end`); + } + } + if (Array.isArray(clip.reframe)) { + clip.reframe.forEach((keyframe, k) => { + if (keyframe.tMs < clip.inMs || keyframe.tMs > clip.outMs) { + fail( + "range_out_of_asset", + `${path}.reframe[${k}].tMs`, + "reframe keyframe lies outside the clip's source range", + ); + } + }); + } + if ( + (profile.kind === "free_social" || context.goal === "teaser") && + context.paidOnly.some( + (segment) => + segment.assetId === clip.assetId && + segment.startMs < clip.outMs && + clip.inMs < segment.endMs, + ) + ) { + fail("paid_only_segment", path, "clip uses material reserved for the paid release"); + } + }); + + // Transitions: known clips, adjacent in time, at most one in and one out per clip. + const blendBetween = new Map(); + const outgoing = new Set(); + const incoming = new Set(); + const order = sortedVideoClips(edl); + const nextOf = new Map(); + for (let i = 0; i + 1 < order.length; i++) nextOf.set(order[i]!.id, order[i + 1]!.id); + edl.transitions.forEach((transition, index) => { + const path = `transitions[${index}]`; + const from = clipsById.get(transition.fromClipId); + const to = clipsById.get(transition.toClipId); + if (!from) fail("unknown_clip", `${path}.fromClipId`, `no clip ${transition.fromClipId}`); + if (!to) fail("unknown_clip", `${path}.toClipId`, `no clip ${transition.toClipId}`); + if (!from || !to) return; + if (nextOf.get(from.id) !== to.id) { + fail("schema", path, "a transition must join a clip to the clip that follows it"); + return; + } + if (outgoing.has(from.id) || incoming.has(to.id)) { + fail("schema", path, "a clip can have only one transition on each side"); + return; + } + outgoing.add(from.id); + incoming.add(to.id); + if (BLENDED_TRANSITIONS.has(transition.kind) && transition.durationMs > 0) { + const shortest = Math.min(clipLengthMs(from), clipLengthMs(to)); + if (transition.durationMs >= shortest) { + fail("overlap", `${path}.durationMs`, "transition is longer than one of its clips"); + return; + } + blendBetween.set(`${from.id}>${to.id}`, transition.durationMs); + } + }); + + // Overlaps on the video track: only blended transitions may overlap, by exactly their length. + for (let i = 0; i + 1 < order.length; i++) { + const a = order[i]!; + const b = order[i + 1]!; + const overlap = a.atMs + clipLengthMs(a) - b.atMs; + const blend = blendBetween.get(`${a.id}>${b.id}`); + const path = `tracks.video[${edl.tracks.video.indexOf(b)}].atMs`; + if (blend !== undefined) { + if (Math.abs(overlap - blend) > 1) { + fail( + "overlap", + path, + `clips ${a.id} and ${b.id} must overlap by the ${blend} ms transition`, + ); + } + } else if (overlap > 1) { + fail("overlap", path, `clip ${b.id} starts ${overlap} ms before ${a.id} ends`); + } + } + for (let i = 0; i + 2 < order.length; i++) { + const a = order[i]!; + const c = order[i + 2]!; + if (a.atMs + clipLengthMs(a) > c.atMs + 1) { + fail("overlap", `tracks.video[${edl.tracks.video.indexOf(c)}].atMs`, "three clips overlap"); + } + } + + // Audio clips. + const audioByKind = new Map>(); + edl.tracks.audio.forEach((clip, index) => { + const path = `tracks.audio[${index}]`; + claimId(clip.id, `${path}.id`); + if (clip.outMs <= clip.inMs) fail("range_empty", path, "outMs must be after inMs"); + if (!context.artifactIds.has(clip.artifactId)) { + fail( + "unknown_artifact", + `${path}.artifactId`, + `artifact ${clip.artifactId} is not available`, + ); + } + if (clip.fadeInMs + clip.fadeOutMs > clip.outMs - clip.inMs && clip.outMs > clip.inMs) { + fail("range_empty", path, "fades are longer than the clip"); + } + if (clip.kind === "music") { + if (profile.audio.music === "platform_library_only") { + fail( + "music_license", + `${path}.artifactId`, + "this platform only allows music added from its own library", + ); + } else if (profile.audio.music === "licensed_only") { + const license = context.music.get(clip.artifactId); + if (!license?.commercialUse) { + fail( + "music_license", + `${path}.artifactId`, + license ? "music is not licensed for commercial use" : "music license is unknown", + ); + } + } + } + if (clip.kind !== "sfx") { + const list = audioByKind.get(clip.kind) ?? []; + list.push({ start: clip.atMs, end: clip.atMs + (clip.outMs - clip.inMs), index }); + audioByKind.set(clip.kind, list); + } + }); + for (const list of audioByKind.values()) { + list.sort((a, b) => a.start - b.start || a.index - b.index); + for (let i = 0; i + 1 < list.length; i++) { + if (list[i]!.end > list[i + 1]!.start) { + fail("overlap", `tracks.audio[${list[i + 1]!.index}].atMs`, "audio clips overlap"); + } + } + } + + // Templates. + const checkTemplate = (templateId: string, path: string) => { + if (!context.templateIds.has(templateId)) { + fail("unknown_template", path, `template ${templateId} is not available`); + } + }; + if (edl.intro) checkTemplate(edl.intro.templateId, "intro.templateId"); + if (edl.outro) checkTemplate(edl.outro.templateId, "outro.templateId"); + + // Overlays. + edl.overlays.forEach((overlay, index) => { + const path = `overlays[${index}]`; + claimId(overlay.id, `${path}.id`); + if (overlay.durationMs <= 0) + fail("range_empty", `${path}.durationMs`, "overlay has no duration"); + if (overlay.motion) checkTemplate(overlay.motion.templateId, `${path}.motion.templateId`); + if (overlay.box && !boxInsideSafeZone(overlay.box, profile)) { + fail("overlay_outside_safe_zone", `${path}.box`, "overlay leaves the profile safe zone"); + } + if (overlay.kind === "watermark" && !context.brandKit?.watermark) { + fail("unknown_artifact", path, "the brand kit has no watermark"); + } + if ((overlay.kind === "text" || overlay.kind === "lower_third") && !overlay.text?.trim()) { + if (!overlay.motion) fail("range_empty", `${path}.text`, "text overlay has no text"); + } + }); + + // Captions preset. + const presetId = edl.captions?.presetId; + if (presetId && context.brandKit) { + if (!context.brandKit.captionPresets.some((preset) => preset.id === presetId)) { + fail("unknown_template", "captions.presetId", `caption preset ${presetId} is not in the kit`); + } + } + + // Redactions. + edl.redactions.faces.forEach((redaction, index) => { + const path = `redactions.faces[${index}]`; + claimId(redaction.id, `${path}.id`); + const target = redaction.target; + if ("trackId" in target && !context.faceTrackIds.has(target.trackId)) { + fail("unknown_track", `${path}.target.trackId`, `face track ${target.trackId} is unknown`); + } + if ("personId" in target && !context.personIds.has(target.personId)) { + fail("unknown_person", `${path}.target.personId`, `person ${target.personId} is unknown`); + } + }); + edl.redactions.marks.forEach((mark, index) => { + if (!context.markIds.has(mark.markId)) { + fail("unknown_mark", `redactions.marks[${index}].markId`, `mark ${mark.markId} is unknown`); + } + }); + edl.redactions.regions.forEach((region, index) => { + const path = `redactions.regions[${index}]`; + claimId(region.id, `${path}.id`); + if (!clipsById.has(region.clipId)) { + fail("unknown_clip", `${path}.clipId`, `no clip ${region.clipId}`); + } + }); + + // Duration against both the plan's and the profile's hard bounds. + const duration = edlDurationMs(edl); + const minMs = Math.max(edl.output.durationHardMinMs, profile.duration.hardMinMs); + const maxMs = Math.min(edl.output.durationHardMaxMs, profile.duration.hardMaxMs); + if (duration < minMs) { + fail("duration_below_min", "output", `duration ${duration} ms is below ${minMs} ms`); + } + if (duration > maxMs) { + fail("duration_above_max", "output", `duration ${duration} ms is above ${maxMs} ms`); + } + + return errors.length > 0 ? { ok: false, errors } : { ok: true, edl }; } diff --git a/packages/media/src/gate.ts b/packages/media/src/gate.ts index f3e4610..56d41ea 100644 --- a/packages/media/src/gate.ts +++ b/packages/media/src/gate.ts @@ -2,16 +2,74 @@ import type { AiAlteration, ContentCategory, Edl, + EdlVideoClip, + FaceRedaction, FfprobeReport, GateIssue, GateReport, NormalizedBox, PlatformProfile, } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { CONTENT_CATEGORIES, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { + BLENDED_TRANSITIONS, + boxInsideSafeZone, + clipLengthMs, + edlDurationMs, + locateOutputTimeAll, + matchesAspect, + sortedVideoClips, +} from "./edl.js"; -/** Tags a clean export may carry. Everything else is a metadata leak. */ -export const METADATA_ALLOWED_TAGS: readonly string[] = []; +/** + * Tags a clean export may carry. Everything else is a metadata leak. + * + * - `title`: the only descriptive tag the compiler writes, and only when set. + * - `major_brand`, `minor_version`, `compatible_brands`: the MP4 file-type + * box, always written by the muxer; it names the format, not the author. + * - `handler_name`, `vendor_id`, `language`: per-stream boilerplate the MP4 + * muxer writes ("VideoHandler", "[0][0][0][0]", "und"). + * - `encoder`: the MP4 sample description always names the codec. It is + * allowed only without a version number (bitexact output writes + * "Lavc libx264"); a versioned value such as "Lavf60.16.100" is a leak. + * + * `creation_time`, `location`, `com.apple.quicktime.*`, `make`, `model`, + * `software`, GPS keys and anything unknown always count as leaks. + */ +export const METADATA_ALLOWED_TAGS: readonly string[] = [ + "title", + "major_brand", + "minor_version", + "compatible_brands", + "handler_name", + "vendor_id", + "language", + "encoder", +]; + +const ALLOWED = new Set(METADATA_ALLOWED_TAGS); +/** "Lavf60.16.100", "Lavc60.31", "x264 core 164", "HandBrake 1.7.0". */ +const VERSIONED = /\d+\.\d+|lav[cf]\d|core\s*\d/i; + +/** Tag name without a "format." / "stream.0." style prefix, lower-cased. */ +function tagName(key: string): string { + return key + .trim() + .toLowerCase() + .replace(/^(?:format|streams?(?:[.:/]\d+)?)[.:/]/, ""); +} + +export function findMetadataLeaks(tags: Record): string[] { + return Object.entries(tags) + .filter(([key, value]) => { + const name = tagName(key); + if (!ALLOWED.has(name)) return true; + if (name === "encoder") return VERSIONED.test(value); + return false; + }) + .map(([key]) => key) + .sort(); +} export interface GateInput { profile: PlatformProfile; @@ -44,28 +102,370 @@ export interface GateInput { now: Date; } -/** Deterministic gate (spec 6). Any blocking issue fails the report. */ -export function evaluateGate(_input: GateInput): GateReport { - return notImplemented("evaluateGate"); +/** Longest tracking gap the anonymizer may bridge (spec 7). */ +export const MAX_TRACKING_GAP_MS = 500; +/** Residual visibility of a treated mark above which the gate blocks. */ +export const MAX_MARK_RESIDUAL = 0.05; +/** Loudness tolerance around the target, in LU. */ +export const LOUDNESS_TOLERANCE_LU = 2; +const TRUE_PEAK_TOLERANCE_DB = 0.5; + +const ms = (value: number) => Math.max(0, Math.round(value)); + +/** Merge time-sorted hits into ranges when they are at most `gapMs` apart. */ +function ranges(hits: T[], gapMs: number): T[][] { + const sorted = [...hits].sort((a, b) => a.tMs - b.tMs); + const out: T[][] = []; + for (const hit of sorted) { + const last = out.at(-1); + if (last && hit.tMs - last.at(-1)!.tMs <= gapMs) last.push(hit); + else out.push([hit]); + } + return out; } export function checkTechnical( - _profile: PlatformProfile, - _edl: Edl, - _probe: FfprobeReport, + profile: PlatformProfile, + edl: Edl, + probe: FfprobeReport, ): GateIssue[] { - return notImplemented("checkTechnical"); + const issues: GateIssue[] = []; + const block = (code: string, message: string) => + issues.push({ check: "technical", severity: "block", code, message }); + if (probe.width !== edl.output.width || probe.height !== edl.output.height) { + block( + "resolution_mismatch", + `output is ${probe.width}x${probe.height}, plan expects ${edl.output.width}x${edl.output.height}`, + ); + } + if (!matchesAspect(probe.width, probe.height, profile.video.aspectRatios)) { + block("aspect_not_allowed", `${probe.width}x${probe.height} is not an allowed aspect ratio`); + } + if (Math.max(probe.width, probe.height) > Math.max(profile.video.width, profile.video.height)) { + block("resolution_too_high", "output is larger than the profile allows"); + } + if ( + Math.abs(probe.fps - edl.output.fps) > edl.output.fps * 0.01 || + probe.fps > profile.video.fps * 1.001 + ) { + block("fps_mismatch", `output runs at ${probe.fps} fps`); + } + const frameMs = 1000 / Math.max(1, edl.output.fps); + const expected = edlDurationMs(edl); + if (probe.durationMs < profile.duration.hardMinMs - frameMs) { + block("duration_too_short", `${probe.durationMs} ms is below ${profile.duration.hardMinMs} ms`); + } + if (probe.durationMs > profile.duration.hardMaxMs + frameMs) { + block("duration_too_long", `${probe.durationMs} ms is above ${profile.duration.hardMaxMs} ms`); + } + if (Math.abs(probe.durationMs - expected) > Math.max(2 * frameMs, 100)) { + issues.push({ + check: "technical", + severity: "warn", + code: "duration_drift", + message: `output is ${probe.durationMs} ms, plan is ${expected} ms`, + }); + } + if (probe.sizeBytes > profile.video.maxFileSizeMb * 1_000_000) { + block("file_too_large", `file is larger than ${profile.video.maxFileSizeMb} MB`); + } + const codec = probe.videoCodec.toLowerCase(); + const expectedCodec = profile.video.codec === "hevc" ? ["hevc", "h265"] : ["h264", "avc1"]; + if (!expectedCodec.includes(codec)) { + block("codec_mismatch", `video codec ${probe.videoCodec} is not ${profile.video.codec}`); + } + if (probe.audioCodec !== null && probe.audioCodec.toLowerCase() !== "aac") { + issues.push({ + check: "technical", + severity: "warn", + code: "audio_codec", + message: `audio codec ${probe.audioCodec} is not AAC`, + }); + } + const target = edl.targetLufs ?? profile.audio.targetLufs; + if (probe.audioCodec !== null) { + if (probe.integratedLufs === null) { + block("loudness_unmeasured", "loudness was not measured"); + } else if (Math.abs(probe.integratedLufs - target) > LOUDNESS_TOLERANCE_LU) { + block("loudness_off_target", `${probe.integratedLufs} LUFS, target ${target} LUFS`); + } + if ( + probe.truePeakDb !== null && + probe.truePeakDb > profile.audio.truePeakDb + TRUE_PEAK_TOLERANCE_DB + ) { + block("true_peak_exceeded", `true peak ${probe.truePeakDb} dBTP`); + } + } + const leaks = findMetadataLeaks(probe.tags); + if (leaks.length > 0) block("metadata_leak", `metadata not removed: ${leaks.join(", ")}`); + return issues; } -export function findMetadataLeaks(_tags: Record): string[] { - return notImplemented("findMetadataLeaks"); +const URL_PATTERN = + /\b(?:https?:\/\/|www\.)\S+|\b[a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:com|net|org|io|de|at|ch|eu|ly|co|me|link|shop|store|app|tv|gg|xyz)\b(?:\/\S*)?/i; +const HASHTAG_PATTERN = /#[\p{L}\p{N}_]+/gu; + +/** Profile patterns compile case-insensitively; an invalid one is reported, never thrown. */ +function compilePattern(pattern: string): RegExp | null { + try { + return new RegExp(pattern, "i"); + } catch { + return null; + } } export function checkCopy( - _profile: PlatformProfile, - _copy: { title: string | null; caption: string | null; tags: string[] }, + profile: PlatformProfile, + copy: { title: string | null; caption: string | null; tags: string[] }, ): GateIssue[] { - return notImplemented("checkCopy"); + const issues: GateIssue[] = []; + const block = (code: string, message: string) => + issues.push({ check: "text", severity: "block", code, message }); + const { text } = profile; + const title = copy.title ?? ""; + const caption = copy.caption ?? ""; + if (text.titleMaxChars !== undefined && [...title].length > text.titleMaxChars) { + block( + "title_too_long", + `title has ${[...title].length} characters, limit ${text.titleMaxChars}`, + ); + } + if ([...caption].length > text.captionMaxChars) { + block( + "caption_too_long", + `caption has ${[...caption].length} characters, limit ${text.captionMaxChars}`, + ); + } + const hashtags = new Set(); + for (const tag of copy.tags) { + const clean = tag.replace(/^#+/, "").trim().toLowerCase(); + if (clean) hashtags.add(clean); + } + for (const match of caption.matchAll(HASHTAG_PATTERN)) + hashtags.add(match[0].slice(1).toLowerCase()); + if (hashtags.size > text.hashtagLimit) { + block("too_many_hashtags", `${hashtags.size} hashtags, limit ${text.hashtagLimit}`); + } + const fields: Array<[string, string]> = [ + ["title", title], + ["caption", caption], + ...copy.tags.map((tag, i): [string, string] => [`tags[${i}]`, tag]), + ]; + for (const banned of text.bannedPatterns) { + const regex = compilePattern(banned.pattern); + if (!regex) { + block("invalid_banned_pattern", `profile pattern cannot be checked: ${banned.reason}`); + continue; + } + for (const [field, value] of fields) { + if (value && regex.test(value)) block("banned_pattern", `${field}: ${banned.reason}`); + } + } + if (text.linkPolicy !== "allowed") { + for (const [field, value] of fields) { + if (!value || !URL_PATTERN.test(value)) continue; + if (text.linkPolicy === "forbidden") block("link_forbidden", `${field} contains a link`); + else block("link_bio_only", `${field} contains a link; links belong in the bio`); + } + } + return issues; +} + +function checkContent(input: GateInput): GateIssue[] { + const { profile } = input; + const issues: GateIssue[] = []; + const thresholds = profile.content.thresholds; + const gapMs = (1000 / profile.content.sampleFps) * 1.5; + const sampled = Math.max(input.content.length, input.sampledFrames, 1); + for (const category of CONTENT_CATEGORIES) { + const threshold = thresholds[category]; + if (threshold === undefined) continue; + const hits = input.content.flatMap((sample) => { + const score = sample.scores[category]; + return score !== undefined && score > threshold ? [{ tMs: sample.tMs, score }] : []; + }); + if (hits.length === 0) continue; + const blocking = + profile.content.blockOn === "any_frame" || + hits.length / sampled > (profile.content.blockRatio ?? 0); + for (const range of ranges(hits, gapMs)) { + const score = Math.max(...range.map((h) => h.score)); + issues.push({ + check: "content", + severity: blocking ? "block" : "warn", + code: "content_threshold", + message: `${category} ${score.toFixed(2)} above ${threshold}`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + category, + score, + }); + } + } + return issues; +} + +function checkAnonymization(input: GateInput): GateIssue[] { + const issues: GateIssue[] = []; + const protectedIds = new Set( + input.people.filter((p) => p.anonymizeByDefault !== "none").map((p) => p.id), + ); + const gapMs = 1000; + const uncovered = input.faces.filter( + (face) => face.personId !== null && protectedIds.has(face.personId) && face.coverage < 1, + ); + const byPerson = new Map(); + for (const face of uncovered) { + const list = byPerson.get(face.personId!) ?? []; + list.push(face); + byPerson.set(face.personId!, list); + } + for (const [personId, faces] of [...byPerson.entries()].sort((a, b) => + a[0].localeCompare(b[0]), + )) { + for (const range of ranges(faces, gapMs)) { + const coverage = Math.min(...range.map((f) => f.coverage)); + issues.push({ + check: "anonymization", + severity: "block", + code: "face_not_covered", + message: `face covered ${(coverage * 100).toFixed(0)} %`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + personId, + score: coverage, + }); + } + } + if (protectedIds.size > 0) { + const unknown = input.faces.filter((face) => face.personId === null); + for (const range of ranges(unknown, gapMs)) { + issues.push({ + check: "anonymization", + severity: "block", + code: "unknown_face", + message: "a face that cannot be matched appears next to a protected person", + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + }); + } + } + for (const gap of input.trackingGaps) { + if (gap.endMs - gap.startMs <= MAX_TRACKING_GAP_MS) continue; + issues.push({ + check: "anonymization", + severity: "block", + code: "tracking_gap", + message: `face tracking lost for ${Math.round(gap.endMs - gap.startMs)} ms (needs review)`, + atMs: ms(gap.startMs), + endMs: ms(gap.endMs), + ...(gap.personId ? { personId: gap.personId } : {}), + }); + } + const visibleMarks = input.marks.filter((mark) => mark.residualVisibility > MAX_MARK_RESIDUAL); + const byMark = new Map(); + for (const mark of visibleMarks) { + const list = byMark.get(mark.markId) ?? []; + list.push(mark); + byMark.set(mark.markId, list); + } + for (const [markId, marks] of [...byMark.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + for (const range of ranges(marks, gapMs)) { + const residual = Math.max(...range.map((m) => m.residualVisibility)); + issues.push({ + check: "anonymization", + severity: "block", + code: "mark_visible", + message: `treated mark still ${(residual * 100).toFixed(0)} % visible`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + markId, + score: residual, + }); + } + } + return issues; +} + +function checkConsent(input: GateInput): GateIssue[] { + const { consent } = input.profile; + if (!consent.requirePerformerConsent && !consent.requireAgeVerification) return []; + const issues: GateIssue[] = []; + for (const person of [...input.people].sort((a, b) => a.id.localeCompare(b.id))) { + if (consent.requirePerformerConsent && !person.consentRecordedAt) { + issues.push({ + check: "consent", + severity: "block", + code: "consent_missing", + message: "no recorded consent", + personId: person.id, + }); + } + if (consent.requireAgeVerification && !person.ageVerifiedAt) { + issues.push({ + check: "consent", + severity: "block", + code: "age_unverified", + message: "age is not verified", + personId: person.id, + }); + } + } + for (const range of ranges( + input.faces.filter((face) => face.personId === null), + 1000, + )) { + issues.push({ + check: "consent", + severity: "block", + code: "unknown_performer", + message: "a visible face is not assigned to a person with consent", + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + }); + } + return issues; +} + +/** Deterministic gate (spec 6). Any blocking issue fails the report. */ +export function evaluateGate(input: GateInput): GateReport { + const { profile } = input; + const issues: GateIssue[] = [ + ...checkTechnical(profile, input.edl, input.ffprobe), + ...checkContent(input), + ...checkAnonymization(input), + ...checkConsent(input), + ...checkCopy(profile, input.copy), + ]; + const alterations = new Set([...input.aiAlterations, ...input.edl.metadata.aiAlterations]); + if ( + alterations.size > 0 && + profile.disclosure.aiAlteredLabelRequired && + !input.bundleHasDisclosure + ) { + issues.push({ + check: "disclosure", + severity: "block", + code: "ai_label_missing", + message: `AI alterations (${[...alterations].sort().join(", ")}) need the platform label`, + }); + } + if (input.watermarkBox && !boxInsideSafeZone(input.watermarkBox, profile)) { + issues.push({ + check: "brand", + severity: "warn", + code: "watermark_outside_safe_zone", + message: "watermark is covered by platform interface", + }); + } + return { + profileId: profile.id, + passed: !issues.some((issue) => issue.severity === "block"), + checkedAt: input.now.toISOString(), + sampledFrames: Math.max(0, Math.round(input.sampledFrames)), + issues, + autoFixRound: Math.min(MAX_AUTO_FIX_ROUNDS, Math.max(0, Math.round(input.autoFixRound))), + }; } export interface AutoFixResult { @@ -73,15 +473,227 @@ export interface AutoFixResult { changes: string[]; } +const PADDING_STEP = 25; +const CUT_MARGIN_MS = 500; +const MIN_KEEP_MS = 500; + +/** Lay clips out back to back again, keeping blended transitions' overlap. */ +function repack(edl: Edl): Edl { + const clips = sortedVideoClips(edl); + const ids = new Set(clips.map((c) => c.id)); + const transitions = edl.transitions.filter((t) => ids.has(t.fromClipId) && ids.has(t.toClipId)); + const adjacent = new Set(); + for (let i = 0; i + 1 < clips.length; i++) adjacent.add(`${clips[i]!.id}>${clips[i + 1]!.id}`); + const kept = transitions.filter((t) => adjacent.has(`${t.fromClipId}>${t.toClipId}`)); + const blend = new Map( + kept + .filter((t) => BLENDED_TRANSITIONS.has(t.kind) && t.durationMs > 0) + .map((t) => [`${t.fromClipId}>${t.toClipId}`, t.durationMs]), + ); + let cursor = 0; + const packed: EdlVideoClip[] = clips.map((clip, i) => { + const previous = clips[i - 1]; + let overlap = previous ? (blend.get(`${previous.id}>${clip.id}`) ?? 0) : 0; + overlap = Math.min(overlap, clipLengthMs(clip) - 1, cursor); + const atMs = Math.max(0, cursor - Math.max(0, overlap)); + cursor = atMs + clipLengthMs(clip); + return { ...clip, atMs }; + }); + const regions = edl.redactions.regions.filter((r) => ids.has(r.clipId)); + const next = { + ...edl, + tracks: { ...edl.tracks, video: packed }, + transitions: kept.filter((t) => { + if (!BLENDED_TRANSITIONS.has(t.kind)) return true; + const from = packed.find((c) => c.id === t.fromClipId)!; + const to = packed.find((c) => c.id === t.toClipId)!; + return t.durationMs < Math.min(clipLengthMs(from), clipLengthMs(to)); + }), + redactions: { ...edl.redactions, regions }, + }; + // Overlays never grow or move; they are only clamped to the new end. + const total = edlDurationMs(next); + next.overlays = edl.overlays.flatMap((overlay) => { + if (overlay.atMs >= total) return []; + return [{ ...overlay, durationMs: Math.min(overlay.durationMs, total - overlay.atMs) }]; + }); + return next; +} + /** * Compliance-only repairs: trim or drop offending clips, change crop, widen * redaction padding, swap to another segment. Never adds obfuscation aimed at * classifiers. Returns null when the round limit is reached or nothing safe applies. + * + * - Content blocks and tracking gaps: the clip showing that moment is swapped + * for an alternative segment of at least the same length that does not + * overlap the offending source range, else trimmed (when the range sits at + * one end) or dropped. The timeline is then repacked. + * - Uncovered faces: padding of the person's face redaction grows by 25 + * points; a missing redaction is added as blur. + * - Unknown faces: the unassigned-face redaction grows or is added as blur. + * - Visible marks: the mark treatment falls back to cover. */ export function planAutoFix( - _edl: Edl, - _report: GateReport, - _alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>, + edl: Edl, + report: GateReport, + alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>, ): AutoFixResult | null { - return notImplemented("planAutoFix"); + if (report.autoFixRound >= MAX_AUTO_FIX_ROUNDS || report.passed) return null; + const blocks = report.issues.filter((issue) => issue.severity === "block"); + const changes: string[] = []; + let clips: EdlVideoClip[] = edl.tracks.video.map((clip) => ({ ...clip })); + let faces: FaceRedaction[] = edl.redactions.faces.map((f) => ({ ...f })); + let marks = edl.redactions.marks.map((m) => ({ ...m })); + let clipsChanged = false; + + // Source ranges to remove, per clip. + const offending = new Map>(); + for (const issue of blocks) { + const timeline = + issue.check === "content" || + issue.code === "tracking_gap" || + (issue.code === "mark_visible" && + !edl.redactions.marks.some((m) => m.markId === issue.markId)); + if (!timeline || issue.atMs === undefined) continue; + const endMs = issue.endMs ?? issue.atMs; + const step = Math.max(1, Math.floor(1000 / edl.output.fps)); + const times: number[] = []; + for (let t = issue.atMs; t < endMs; t += step) times.push(t); + times.push(endMs); + for (const t of times) { + for (const hit of locateOutputTimeAll(edl, t)) { + const list = offending.get(hit.clipId) ?? []; + const last = list.at(-1); + if (last && last.reason === issue.code && hit.sourceMs - last.end <= step * 4) { + last.end = Math.max(last.end, hit.sourceMs); + } else { + list.push({ start: hit.sourceMs, end: hit.sourceMs, reason: issue.code }); + } + offending.set(hit.clipId, list); + } + } + } + const usedAlternatives = new Set(); + for (const [clipId, found] of [...offending.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + const clip = clips.find((c) => c.id === clipId); + if (!clip) continue; + const start = Math.max(clip.inMs, Math.min(...found.map((r) => r.start)) - CUT_MARGIN_MS); + const end = Math.min(clip.outMs, Math.max(...found.map((r) => r.end)) + CUT_MARGIN_MS); + const length = clip.outMs - clip.inMs; + const alternative = alternatives + .filter( + (a) => + !usedAlternatives.has(a.segmentId) && + a.segmentId !== clip.segmentId && + a.endMs - a.startMs >= length && + !(a.assetId === clip.assetId && a.startMs < end && start < a.endMs), + ) + .sort((a, b) => a.segmentId.localeCompare(b.segmentId))[0]; + clipsChanged = true; + if (alternative) { + usedAlternatives.add(alternative.segmentId); + Object.assign(clip, { + assetId: alternative.assetId, + segmentId: alternative.segmentId, + inMs: alternative.startMs, + outMs: alternative.startMs + length, + reframe: clip.reframe === "none" ? "none" : "auto", + }); + changes.push( + `clip ${clip.id}: swapped to segment ${alternative.segmentId} (${found[0]!.reason})`, + ); + } else if (start - clip.inMs >= MIN_KEEP_MS && end >= clip.outMs - 1) { + clip.outMs = start; + changes.push(`clip ${clip.id}: trimmed the end at ${start} ms (${found[0]!.reason})`); + } else if (clip.outMs - end >= MIN_KEEP_MS && start <= clip.inMs + 1) { + clip.inMs = end; + changes.push(`clip ${clip.id}: trimmed the start to ${end} ms (${found[0]!.reason})`); + } else { + clips = clips.filter((c) => c.id !== clip.id); + changes.push(`clip ${clip.id}: removed (${found[0]!.reason})`); + } + } + for (const clip of clips) { + if (Array.isArray(clip.reframe)) { + const inside = clip.reframe.filter((k) => k.tMs >= clip.inMs && k.tMs <= clip.outMs); + clip.reframe = inside.length > 0 ? inside : "auto"; + } + } + + const widen = (match: (f: FaceRedaction) => boolean, add: () => FaceRedaction, label: string) => { + const existing = faces.filter(match); + if (existing.length === 0) { + faces = [...faces, add()]; + changes.push(`${label}: added blur redaction`); + return; + } + for (const face of existing) { + if (face.paddingPct >= 100) continue; + face.paddingPct = Math.min(100, face.paddingPct + PADDING_STEP); + changes.push(`${label}: redaction ${face.id} padding ${face.paddingPct} %`); + } + }; + const widened = new Set(); + for (const issue of blocks) { + if (issue.code === "face_not_covered" && issue.personId && !widened.has(issue.personId)) { + const personId = issue.personId; + widened.add(personId); + widen( + (f) => "personId" in f.target && f.target.personId === personId, + () => ({ + id: `fix-face-${widened.size}`, + target: { personId }, + mode: "blur", + paddingPct: 20 + PADDING_STEP, + featherPct: 10, + minSizePct: 2, + }), + `person ${personId}`, + ); + } + if (issue.code === "unknown_face" && !widened.has("(unassigned)")) { + widened.add("(unassigned)"); + widen( + (f) => "unassignedFaces" in f.target, + () => ({ + id: "fix-face-unassigned", + target: { unassignedFaces: true }, + mode: "blur", + paddingPct: 20 + PADDING_STEP, + featherPct: 10, + minSizePct: 2, + }), + "unassigned faces", + ); + } + if (issue.code === "mark_visible" && issue.markId) { + const markId = issue.markId; + marks = marks.map((mark) => { + if (mark.markId !== markId || mark.treatment === "cover") return mark; + changes.push(`mark ${markId}: ${mark.treatment} replaced by cover`); + return { ...mark, treatment: "cover" as const }; + }); + } + } + if (changes.length === 0 || clips.length === 0) return null; + + let next: Edl = { + ...edl, + tracks: { ...edl.tracks, video: clips }, + redactions: { ...edl.redactions, faces, marks }, + }; + if (clipsChanged) next = repack(next); + const stillAltered = new Set( + next.metadata.aiAlterations.filter( + (a) => a !== "inpaint" || marks.some((m) => m.treatment === "inpaint"), + ), + ); + const explicit = edl.metadata.aiAltered && edl.metadata.aiAlterations.length === 0; + next.metadata = { + ...next.metadata, + aiAlterations: [...stillAltered], + aiAltered: stillAltered.size > 0 || explicit, + }; + return { edl: next, changes }; } diff --git a/packages/media/src/marketing.ts b/packages/media/src/marketing.ts index e63db11..2a0ee0c 100644 --- a/packages/media/src/marketing.ts +++ b/packages/media/src/marketing.ts @@ -6,15 +6,70 @@ import type { PlatformProfile, Utm, } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; -/** Append UTM parameters to an http(s) target. Rejects other schemes. */ -export function buildTrackingUrl(_targetUrl: string, _utm: Utm): string { - return notImplemented("buildTrackingUrl"); +const UTM_KEYS: Array<[keyof Utm, string]> = [ + ["source", "utm_source"], + ["medium", "utm_medium"], + ["campaign", "utm_campaign"], + ["content", "utm_content"], + ["term", "utm_term"], +]; + +/** + * Append UTM parameters to an http(s) target. Rejects other schemes. Existing + * query parameters keep their exact encoding; existing utm_* values for the + * keys being set are replaced, and the fragment stays last. + */ +export function buildTrackingUrl(targetUrl: string, utm: Utm): string { + let url: URL; + try { + url = new URL(targetUrl.trim()); + } catch { + throw new Error("tracking target is not a valid URL"); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("tracking target must use http or https"); + } + const setting = UTM_KEYS.filter(([key]) => utm[key]?.trim()); + const replaced = new Set(setting.map(([, name]) => name)); + const kept = url.search + .replace(/^\?/, "") + .split("&") + .filter((part) => { + if (!part) return false; + const rawKey = part.split("=")[0] ?? ""; + let key = rawKey; + try { + key = decodeURIComponent(rawKey.replace(/\+/g, " ")); + } catch { + // keep undecodable keys as they are + } + return !replaced.has(key); + }); + const added = setting.map( + ([key, name]) => `${name}=${encodeURIComponent((utm[key] as string).trim())}`, + ); + const query = [...kept, ...added].join("&"); + return `${url.origin}${url.pathname}${query ? `?${query}` : ""}${url.hash}`; } -export function normalizeHashtags(_tags: string[], _limit: number): string[] { - return notImplemented("normalizeHashtags"); +/** + * Normalize to "#tag": strip leading #, drop spaces and punctuation (letters, + * digits and _ stay, in any script), dedupe case-insensitively, and cap. + */ +export function normalizeHashtags(tags: string[], limit: number): string[] { + const seen = new Set(); + const out: string[] = []; + for (const raw of tags) { + const tag = raw.normalize("NFC").replace(/[^\p{L}\p{N}_]/gu, ""); + if (!tag) continue; + const key = tag.toLocaleLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + out.push(`#${tag}`); + if (out.length >= Math.max(0, limit)) break; + } + return out.slice(0, Math.max(0, limit)); } export interface ParsedMetricRow { @@ -28,39 +83,450 @@ export interface ParsedMetricRow { currency: string; } -/** RFC 4180 CSV parsing with the saved column mapping. */ +/** RFC 4180 records; quoted fields may hold delimiters, quotes ("") and newlines. */ +export function parseCsvRecords(csv: string, delimiter: string): string[][] { + const text = csv.replace(/^/, ""); + const records: string[][] = []; + let record: string[] = []; + let field = ""; + let quoted = false; + let i = 0; + while (i < text.length) { + const char = text[i]!; + if (quoted) { + if (char === '"') { + if (text[i + 1] === '"') { + field += '"'; + i += 2; + continue; + } + quoted = false; + i += 1; + continue; + } + field += char; + i += 1; + continue; + } + if (char === '"' && field.length === 0) { + quoted = true; + i += 1; + } else if (char === delimiter) { + record.push(field); + field = ""; + i += 1; + } else if (char === "\r" || char === "\n") { + record.push(field); + records.push(record); + record = []; + field = ""; + i += char === "\r" && text[i + 1] === "\n" ? 2 : 1; + } else { + field += char; + i += 1; + } + } + if (field.length > 0 || record.length > 0) { + record.push(field); + records.push(record); + } + return records; +} + +const CURRENCY_SYMBOLS: Record = { "€": "EUR", $: "USD", "£": "GBP", "¥": "JPY" }; + +/** Whole, non-negative count with optional thousands separators. */ +function parseCount(raw: string): number | null { + const value = raw.trim().replace(/[\s  ']/g, ""); + if (value === "") return 0; + if (/^\d+$/.test(value)) return Number(value); + if (/^\d{1,3}([.,]\d{3})+$/.test(value) && new Set(value.match(/[.,]/g)).size === 1) { + return Number(value.replace(/[.,]/g, "")); + } + return null; +} + +/** + * Money in major units to cents. With both "," and "." the later one is the + * decimal separator; a single separator followed by one or two digits is + * decimal, by exactly three digits after 1–999 it groups thousands ("1,234"), + * otherwise it is decimal ("0.995"). Extra decimals round half up. + */ +export function parseMoneyToCents(raw: string): { cents: number; currency: string | null } | null { + let value = raw.trim(); + let currency: string | null = null; + for (const [symbol, code] of Object.entries(CURRENCY_SYMBOLS)) { + if (value.includes(symbol)) { + currency = code; + value = value.replace(symbol, ""); + } + } + const code = value.match(/[A-Za-z]{3}/)?.[0]; + if (code) { + currency = code.toUpperCase(); + value = value.replace(code, ""); + } + value = value.replace(/[\s  ']/g, ""); + if (value === "") return { cents: 0, currency }; + if (!/^\d[\d.,]*$/.test(value)) return null; + const lastComma = value.lastIndexOf(","); + const lastDot = value.lastIndexOf("."); + let integer = value; + let fraction = ""; + if (lastComma >= 0 && lastDot >= 0) { + const decimal = Math.max(lastComma, lastDot); + integer = value.slice(0, decimal); + fraction = value.slice(decimal + 1); + } else if (lastComma >= 0 || lastDot >= 0) { + const separator = lastComma >= 0 ? "," : "."; + const parts = value.split(separator); + const tail = parts.at(-1)!; + const decimal = + parts.length === 2 && + tail.length >= 1 && + (tail.length <= 2 || !/^[1-9]\d{0,2}$/.test(parts[0]!)); + if (decimal) { + integer = parts[0]!; + fraction = tail; + } else if (parts.slice(1).every((part) => part.length === 3)) { + integer = parts.join(""); + } else { + return null; + } + } + integer = integer.replace(/[.,]/g, ""); + if (!/^\d+$/.test(integer) || !/^\d*$/.test(fraction)) return null; + const twoDigits = `${fraction}00`.slice(0, 2); + const roundUp = fraction.length > 2 && Number(fraction[2]) >= 5 ? 1 : 0; + return { cents: Number(integer) * 100 + Number(twoDigits) + roundUp, currency }; +} + +/** ISO dates and times, YYYY/MM/DD, and DD.MM.YYYY. Ambiguous slash dates are rejected. */ +function parseDate(raw: string): Date | null { + const value = raw.trim(); + let match = value.match(/^(\d{4})[-/](\d{2})[-/](\d{2})$/); + if (match) return utcDate(Number(match[1]), Number(match[2]), Number(match[3])); + match = value.match(/^(\d{1,2})\.(\d{1,2})\.(\d{4})$/); + if (match) return utcDate(Number(match[3]), Number(match[2]), Number(match[1])); + if (/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(value)) { + const date = new Date(value.replace(" ", "T")); + return Number.isNaN(date.getTime()) ? null : date; + } + return null; +} + +function utcDate(year: number, month: number, day: number): Date | null { + const date = new Date(Date.UTC(year, month - 1, day)); + return date.getUTCFullYear() === year && + date.getUTCMonth() === month - 1 && + date.getUTCDate() === day + ? date + : null; +} + +/** RFC 4180 CSV parsing with the saved column mapping. Rows count from the header as row 1. */ export function parseMetricsCsv( - _csv: string, - _mapping: MetricCsvMapping, + csv: string, + mapping: MetricCsvMapping, ): { rows: ParsedMetricRow[]; skipped: Array<{ row: number; reason: string }> } { - return notImplemented("parseMetricsCsv"); + const records = parseCsvRecords(csv, mapping.delimiter ?? ","); + const header = records[0]; + const rows: ParsedMetricRow[] = []; + const skipped: Array<{ row: number; reason: string }> = []; + if (!header) return { rows, skipped: [{ row: 1, reason: "file is empty" }] }; + const names = header.map((name) => name.trim()); + const column = (name: string | undefined): number => { + if (!name) return -1; + const exact = names.indexOf(name.trim()); + if (exact >= 0) return exact; + const lower = name.trim().toLowerCase(); + return names.findIndex((n) => n.toLowerCase() === lower); + }; + const columns = mapping.columns; + const index = { + publication: column(columns.publication), + observedAt: column(columns.observedAt), + views: column(columns.views), + clicks: column(columns.clicks), + conversions: column(columns.conversions), + revenue: column(columns.revenue), + currency: column(columns.currency), + }; + const missing = (Object.keys(index) as Array).filter( + (key) => columns[key] !== undefined && index[key] < 0, + ); + if (missing.length > 0) { + return { + rows, + skipped: [ + { row: 1, reason: `missing columns: ${missing.map((k) => columns[k]).join(", ")}` }, + ], + }; + } + const defaultCurrency = (mapping.defaultCurrency ?? "EUR").toUpperCase(); + + for (const [offset, record] of records.slice(1).entries()) { + const row = offset + 2; + if (record.every((field) => field.trim() === "")) continue; + const cell = (i: number) => (i >= 0 ? (record[i] ?? "") : ""); + const publicationKey = cell(index.publication).trim(); + if (!publicationKey) { + skipped.push({ row, reason: "publication is empty" }); + continue; + } + const observedAt = parseDate(cell(index.observedAt)); + if (!observedAt) { + skipped.push({ row, reason: "date is not recognized" }); + continue; + } + const counts = { + views: parseCount(cell(index.views)), + clicks: parseCount(cell(index.clicks)), + conversions: parseCount(cell(index.conversions)), + }; + const invalid = (Object.keys(counts) as Array).find( + (key) => counts[key] === null, + ); + if (invalid) { + skipped.push({ row, reason: `${invalid} is not a whole number` }); + continue; + } + let revenueCents = 0; + let currency: string | null = null; + const revenueRaw = cell(index.revenue); + if (revenueRaw.trim()) { + if (/^\s*-/.test(revenueRaw)) { + skipped.push({ row, reason: "revenue is negative" }); + continue; + } + if (mapping.revenueUnit === "cents") { + const value = parseCount(revenueRaw); + if (value === null) { + skipped.push({ row, reason: "revenue is not a whole number" }); + continue; + } + revenueCents = value; + } else { + const money = parseMoneyToCents(revenueRaw); + if (!money) { + skipped.push({ row, reason: "revenue is not a number" }); + continue; + } + revenueCents = money.cents; + currency = money.currency; + } + } + const currencyCell = cell(index.currency).trim().toUpperCase(); + if (currencyCell && !/^[A-Z]{3}$/.test(currencyCell)) { + skipped.push({ row, reason: "currency is not a 3-letter code" }); + continue; + } + rows.push({ + row, + publicationKey, + observedAt, + views: counts.views ?? 0, + clicks: counts.clicks ?? 0, + conversions: counts.conversions ?? 0, + revenueCents, + currency: currencyCell || currency || defaultCurrency, + }); + } + return { rows, skipped }; } -/** Whether the brand intro fits before the hook window; shorts put the hook first. */ -export function introFits(_profile: PlatformProfile, _introMs: number): boolean { - return notImplemented("introFits"); +const SHORT_FORM_MAX_MS = 180_000; +const MAX_INTRO_MS = 4000; + +/** Vertical profiles capped at three minutes: the hook must come before any branding. */ +export function isShortForm(profile: PlatformProfile): boolean { + return ( + profile.video.aspectRatios.includes("9:16") && profile.duration.hardMaxMs <= SHORT_FORM_MAX_MS + ); } -/** Deterministic intro proposals from the brand kit, tone, and available templates. */ +/** + * Whether the brand intro may open the video. Short-form vertical profiles + * always put the hook first (the brand appears as watermark or end card). + * Elsewhere the intro may take at most 4 s and a tenth of the maximum length. + */ +export function introFits(profile: PlatformProfile, introMs: number): boolean { + if (introMs <= 0) return true; + if (isShortForm(profile)) return false; + return introMs <= Math.min(MAX_INTRO_MS, profile.duration.hardMaxMs / 10); +} + +const TONE_STYLES: Array<{ pattern: RegExp; keywords: string[] }> = [ + { + pattern: /energ|bold|dynamic|loud|wild|sport|fast|laut|schnell|kraft/i, + keywords: ["glitch", "energy", "kinetic"], + }, + { + pattern: /elegan|calm|luxur|soft|premium|ruhig|edel|sanft|warm/i, + keywords: ["elegant", "fade", "minimal"], + }, + { + pattern: /minimal|clean|simple|schlicht|klar|modern/i, + keywords: ["minimal", "logo", "fade"], + }, + { pattern: /playful|fun|witzig|verspielt|bunt/i, keywords: ["kinetic", "typo", "glitch"] }, +]; + +const PARAM_SOURCES: Record string | undefined> = { + primarycolor: (kit) => kit.colors.primary, + secondarycolor: (kit) => kit.colors.secondary, + accentcolor: (kit) => kit.colors.accent, + textcolor: (kit) => kit.colors.text, + backgroundcolor: (kit) => kit.colors.background, + brandname: (kit) => kit.name, + title: (kit) => kit.name, + name: (kit) => kit.name, + logoartifactid: (kit) => kit.logoArtifactIds[0], + logo: (kit) => kit.logoArtifactIds[0], + fontfamily: (kit) => (kit.fonts.find((f) => f.role === "heading") ?? kit.fonts[0])?.family, + soundartifactid: (kit) => kit.musicBeds.find((bed) => bed.commercialUse)?.artifactId, + sound: (kit) => kit.musicBeds.find((bed) => bed.commercialUse)?.artifactId, +}; + +const VARIANT_DURATIONS = [2500, 2000, 3000, 3500, 1500, 4000]; + +/** + * Deterministic intro proposals from the brand kit, tone, and available + * templates: intro templates are ranked by how well their id and name match + * the tone's style keywords (then by id), each proposal uses a distinct + * template while there are enough, and durations stay within 1.5–4 s and the + * template's own limits. Parameters are filled from the kit by name. + */ export function proposeIntroVariants( - _kit: BrandKit, - _templates: MotionTemplateInfo[], - _count: number, + kit: BrandKit, + templates: MotionTemplateInfo[], + count: number, ): Array<{ template: MotionTemplateRef; rationale: string }> { - return notImplemented("proposeIntroVariants"); + const intros = templates + .filter((t) => t.kind === "intro") + .sort((a, b) => a.id.localeCompare(b.id)); + if (intros.length === 0 || count <= 0) return []; + const tone = `${kit.tone ?? ""} ${kit.audience ?? ""}`; + const keywords = TONE_STYLES.filter((style) => style.pattern.test(tone)).flatMap( + (style) => style.keywords, + ); + const affinity = (template: MotionTemplateInfo) => { + const haystack = `${template.id} ${template.name}`.toLowerCase(); + return keywords.reduce( + (score, keyword, index) => score + (haystack.includes(keyword) ? keywords.length - index : 0), + 0, + ); + }; + const ranked = [...intros].sort((a, b) => affinity(b) - affinity(a) || a.id.localeCompare(b.id)); + + return Array.from({ length: count }, (_, index) => { + const template = ranked[index % ranked.length]!; + const round = Math.floor(index / ranked.length); + const wanted = VARIANT_DURATIONS[(index + round) % VARIANT_DURATIONS.length]!; + const durationMs = Math.min( + Math.max(wanted, 1500, template.minDurationMs), + Math.min(4000, template.maxDurationMs), + ); + const params: Record = {}; + for (const param of template.params) { + const source = PARAM_SOURCES[param.name.toLowerCase().replace(/[^a-z]/g, "")]; + const value = source?.(kit); + if (value !== undefined) params[param.name] = value.slice(0, 200); + } + const matched = affinity(template) > 0 && kit.tone ? ` matches the tone "${kit.tone}"` : ""; + return { + template: { templateId: template.id, params, durationMs: Math.max(0, durationMs) }, + rationale: `${template.name}${matched}, ${(durationMs / 1000).toFixed(1)} s`, + }; + }); } +function luminance(hex: string): number { + const channel = (offset: number) => { + const value = Number.parseInt(hex.slice(offset, offset + 2), 16) / 255; + return value <= 0.03928 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4; + }; + return 0.2126 * channel(1) + 0.7152 * channel(3) + 0.0722 * channel(5); +} + +/** WCAG contrast ratio of two #RRGGBB colours. */ +export function contrastRatio(a: string, b: string): number { + const [light, dark] = [luminance(a), luminance(b)].sort((x, y) => y - x) as [number, number]; + return (light + 0.05) / (dark + 0.05); +} + +const MIN_CONTRAST = 4.5; + /** Structural brand-kit checks beyond the schema (contrast, duplicate presets). */ -export function validateBrandKit(_kit: BrandKit): string[] { - return notImplemented("validateBrandKit"); +export function validateBrandKit(kit: BrandKit): string[] { + const warnings: string[] = []; + const ratio = contrastRatio(kit.colors.text, kit.colors.background); + if (ratio < MIN_CONTRAST) { + warnings.push(`text on background has contrast ${ratio.toFixed(1)}:1, below ${MIN_CONTRAST}:1`); + } + const seen = new Set(); + for (const preset of kit.captionPresets) { + if (seen.has(preset.id)) warnings.push(`caption preset id "${preset.id}" is used twice`); + seen.add(preset.id); + if (preset.backgroundColor) { + const presetRatio = contrastRatio(preset.textColor, preset.backgroundColor); + if (presetRatio < MIN_CONTRAST) { + warnings.push( + `caption preset "${preset.id}" has contrast ${presetRatio.toFixed(1)}:1, below ${MIN_CONTRAST}:1`, + ); + } + } + } + for (const bed of kit.musicBeds) { + if (!bed.commercialUse) + warnings.push(`music bed "${bed.title}" is not licensed for commercial use`); + } + for (const font of kit.fonts) { + if (!font.license?.trim()) warnings.push(`font "${font.family}" has no license note`); + } + if (kit.watermark && !kit.watermark.artifactId) warnings.push("watermark has no image"); + return warnings; } /** Upload checklist markdown for an export bundle. */ -export function uploadChecklist(_input: { +export function uploadChecklist(input: { profile: PlatformProfile; aiAltered: boolean; hasCaptions: boolean; trackingUrl: string | null; }): string { - return notImplemented("uploadChecklist"); + const { profile } = input; + const items: string[] = ["Upload the video file and choose the poster as cover image."]; + if (input.aiAltered && profile.disclosure.aiAlteredLabelRequired) { + items.push( + "Set the platform's AI-generated or altered content label: this video contains AI alterations.", + ); + } + if (profile.disclosure.ageRestrictedFlag) items.push("Mark the upload as age-restricted."); + if (profile.disclosure.paidPartnershipLabel) { + items.push("Turn on the paid partnership label if this post is sponsored."); + } + if (input.trackingUrl) { + const link = `<${input.trackingUrl}>`; + if (profile.text.linkPolicy === "allowed") items.push(`Add the tracking link: ${link}`); + else if (profile.text.linkPolicy === "bio_only") { + items.push(`Put the tracking link in the profile bio, not in the caption: ${link}`); + } else items.push("Do not add links: this platform does not allow them."); + } else if (profile.text.linkPolicy === "forbidden") { + items.push("Do not add links: this platform does not allow them."); + } + if (input.hasCaptions) { + items.push("Upload the SRT or VTT subtitles, or confirm the burned-in captions are readable."); + } else if (profile.captions.required) { + items.push("Captions are required on this platform: add them before publishing."); + } + const limits = [ + profile.text.titleMaxChars ? `title up to ${profile.text.titleMaxChars} characters` : null, + `caption up to ${profile.text.captionMaxChars} characters`, + `at most ${profile.text.hashtagLimit} hashtags`, + ].filter(Boolean); + items.push(`Paste the caption from caption.txt (${limits.join(", ")}).`); + return `# Upload checklist: ${profile.name.replace(/[\r\n]+/g, " ")}\n\n${items + .map((item) => `- [ ] ${item}`) + .join("\n")}\n`; } diff --git a/packages/media/src/not-implemented.ts b/packages/media/src/not-implemented.ts deleted file mode 100644 index 0c4ab30..0000000 --- a/packages/media/src/not-implemented.ts +++ /dev/null @@ -1,4 +0,0 @@ -/** Phase A scaffold marker. No call site may remain once Media Studio ships. */ -export function notImplemented(name: string): never { - throw new Error(`not implemented: ${name}`); -} diff --git a/packages/media/src/teaser.ts b/packages/media/src/teaser.ts index d52418f..e8c7818 100644 --- a/packages/media/src/teaser.ts +++ b/packages/media/src/teaser.ts @@ -1,4 +1,6 @@ import type { + AiAlteration, + AnonymizeMode, BrandKit, Edl, MediaProjectGoal, @@ -6,12 +8,88 @@ import type { TeaserFeature, TeaserWeights, } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { + AnonymizeModeSchema, + EdlSchema, + TEASER_FEATURES, + TEASER_LEARNING_MIN_SAMPLES, +} from "@rakazo/contracts"; +import { introFits } from "./marketing.js"; export type SegmentFeatures = Partial>; -export function defaultTeaserWeights(_goal: MediaProjectGoal): TeaserWeights { - return notImplemented("defaultTeaserWeights"); +const GOAL_WEIGHTS: Record> = { + teaser: { + motion: 0.14, + sharpness: 0.06, + faces: 0.08, + gaze: 0.08, + audioPeak: 0.14, + speechHook: 0.24, + novelty: 0.12, + beatProximity: 0.06, + aesthetics: 0.08, + }, + trailer: { + motion: 0.16, + sharpness: 0.08, + faces: 0.08, + gaze: 0.06, + audioPeak: 0.14, + speechHook: 0.12, + novelty: 0.12, + beatProximity: 0.12, + aesthetics: 0.12, + }, + highlight: { + motion: 0.2, + sharpness: 0.08, + faces: 0.12, + gaze: 0.06, + audioPeak: 0.18, + speechHook: 0.08, + novelty: 0.1, + beatProximity: 0.08, + aesthetics: 0.1, + }, + story: { + motion: 0.08, + sharpness: 0.08, + faces: 0.16, + gaze: 0.12, + audioPeak: 0.06, + speechHook: 0.22, + novelty: 0.12, + beatProximity: 0.04, + aesthetics: 0.12, + }, + full_release: { + motion: 0.1, + sharpness: 0.14, + faces: 0.1, + gaze: 0.08, + audioPeak: 0.08, + speechHook: 0.1, + novelty: 0.12, + beatProximity: 0.06, + aesthetics: 0.22, + }, + custom: { + motion: 0.12, + sharpness: 0.1, + faces: 0.1, + gaze: 0.1, + audioPeak: 0.12, + speechHook: 0.14, + novelty: 0.12, + beatProximity: 0.08, + aesthetics: 0.12, + }, +}; + +/** Default per-goal weights over every teaser feature; each set sums to 1. */ +export function defaultTeaserWeights(goal: MediaProjectGoal): TeaserWeights { + return { ...GOAL_WEIGHTS[goal] }; } export interface ScoredSegmentInput { @@ -25,13 +103,75 @@ export interface ScoredSegmentInput { transcript: string | null; } -export function scoreSegment(_features: SegmentFeatures, _weights: TeaserWeights): number { - return notImplemented("scoreSegment"); +const clamp01 = (value: number) => (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0); + +/** Weighted mean of 0..1 features; missing features count as 0. */ +export function scoreSegment(features: SegmentFeatures, weights: TeaserWeights): number { + let total = 0; + let norm = 0; + for (const feature of TEASER_FEATURES) { + const weight = weights[feature] ?? 0; + norm += Math.abs(weight); + total += weight * clamp01(features[feature] ?? 0); + } + return norm === 0 ? 0 : total / norm; } +const HOOK_LEXICON = { + en: { + questionWords: + /^(what|why|how|who|when|where|which|did|do|does|can|could|would|should|is|are|have|ever)\b/, + announcements: + /\b(wait|watch|look|listen|check this|you won'?t believe|here'?s|secret|how to|never|finally|the moment|let me show|guess|imagine|stop|nobody|everyone|biggest|best|worst|first time|new|today)\b/, + address: /\b(you|your)\b/, + numbers: /\b(one|two|three|five|ten|hundred|thousand|million|first|second|third|top)\b/, + }, + de: { + questionWords: + /^(was|warum|wieso|weshalb|wie|wer|wann|wo|welche[rsmn]?|hast|habt|kannst|könnt|willst|würdest|weißt|ist|sind)\b/, + announcements: + /\b(warte|wartet|schau|schaut|guck|hör zu|pass auf|achtung|du wirst nicht glauben|geheimnis|so geht|niemals|endlich|gleich|hier ist|zeig ich dir|stell dir vor|rate mal|niemand|alle|größte|beste|schlimmste|zum ersten mal|neu|heute)\b/, + address: /\b(du|dich|dir|dein|deine|ihr|euch|euer)\b/, + numbers: /\b(eins|zwei|drei|fünf|zehn|hundert|tausend|million|erste[rsn]?|zweite[rsn]?|top)\b/, + }, +} as const; + /** 0..1 speech-hook strength from a transcript excerpt (questions, exclamations, numbers, announcements). */ -export function speechHookScore(_text: string, _language?: string): number { - return notImplemented("speechHookScore"); +export function speechHookScore(text: string, language?: string): number { + const normalized = text.trim().toLocaleLowerCase(language); + if (!normalized) return 0; + const code = language?.slice(0, 2).toLowerCase(); + const lexicons = + code === "de" + ? [HOOK_LEXICON.de] + : code === "en" + ? [HOOK_LEXICON.en] + : Object.values(HOOK_LEXICON); + let score = 0; + if (normalized.includes("?")) score += 0.35; + if (normalized.includes("!")) score += 0.25; + const hasNumber = + /\d/.test(normalized) || lexicons.some((lexicon) => lexicon.numbers.test(normalized)); + if (hasNumber) score += 0.2; + if (lexicons.some((lexicon) => lexicon.announcements.test(normalized))) score += 0.3; + if (lexicons.some((lexicon) => lexicon.questionWords.test(normalized))) score += 0.15; + if (lexicons.some((lexicon) => lexicon.address.test(normalized))) score += 0.1; + return Math.min(1, Math.round(score * 1000) / 1000); +} + +function segmentFeatures(segment: ScoredSegmentInput): SegmentFeatures { + if (segment.features.speechHook !== undefined || !segment.transcript) return segment.features; + return { ...segment.features, speechHook: speechHookScore(segment.transcript) }; +} + +/** Strength of a segment as an opener: speech hooks, audio peaks, motion, eye contact. */ +function hookStrength(features: SegmentFeatures): number { + return ( + 0.45 * clamp01(features.speechHook ?? 0) + + 0.25 * clamp01(features.audioPeak ?? 0) + + 0.15 * clamp01(features.motion ?? 0) + + 0.15 * clamp01(features.gaze ?? 0) + ); } export interface TeaserSelection { @@ -41,12 +181,20 @@ export interface TeaserSelection { notes: string[]; } +const MIN_CLIP_MS = 1000; + /** * Pick clips for a goal and profile: a hook inside hookWindowMs, rising * tension, cut before the payoff (open loop), never paid-only segments, * total within the profile's duration bounds. + * + * The primary asset is the one holding the best segment overall; its payoff + * is its best segment (paid-only included, since that is usually the payoff). + * Nothing from the primary asset at or after the payoff is used. The hook + * opens the teaser, the rest follows in ascending score, and the lead-in to + * the payoff closes it, leaving `endCardMs` for the end card. */ -export function selectTeaserSegments(_input: { +export function selectTeaserSegments(input: { segments: ScoredSegmentInput[]; profile: PlatformProfile; goal: MediaProjectGoal; @@ -54,19 +202,227 @@ export function selectTeaserSegments(_input: { weights: TeaserWeights; endCardMs: number; }): TeaserSelection { - return notImplemented("selectTeaserSegments"); + const { profile } = input; + const notes: string[] = []; + const scored = input.segments + .filter((s) => s.endMs > s.startMs) + .map((segment) => { + const features = segmentFeatures(segment); + return { segment, features, score: scoreSegment(features, input.weights) }; + }) + .sort( + (a, b) => + b.score - a.score || + a.segment.assetId.localeCompare(b.segment.assetId) || + a.segment.startMs - b.segment.startMs || + a.segment.id.localeCompare(b.segment.id), + ); + const maxMs = Math.max( + 0, + Math.min(input.targetMs, profile.duration.hardMaxMs) - Math.max(0, input.endCardMs), + ); + const minMs = Math.max(0, profile.duration.hardMinMs - Math.max(0, input.endCardMs)); + const usable = scored.filter((s) => !s.segment.paidOnly); + if (usable.length === 0 || maxMs < MIN_CLIP_MS) { + return { clips: [], hookSegmentId: null, notes: ["no usable segments"] }; + } + + const primaryAsset = scored[0]!.segment.assetId; + const payoff = scored.find((s) => s.segment.assetId === primaryAsset)!; + const pool = usable.filter( + (s) => + s.segment.assetId !== primaryAsset || + (s.segment.id !== payoff.segment.id && s.segment.endMs <= payoff.segment.startMs), + ); + const openLoop = pool.length < usable.length || payoff.segment.paidOnly; + if (pool.length === 0) { + return { clips: [], hookSegmentId: null, notes: ["only the payoff is usable"] }; + } + + const hook = [...pool].sort( + (a, b) => + hookStrength(b.features) - hookStrength(a.features) || + b.score - a.score || + a.segment.id.localeCompare(b.segment.id), + )[0]!; + const hookWindow = profile.duration.hookWindowMs; + const clipCap = Math.max(MIN_CLIP_MS, Math.min(5000, Math.round(maxMs / 4))); + const lengthOf = (s: ScoredSegmentInput, cap: number) => Math.min(s.endMs - s.startMs, cap); + + // Lead-in: the last primary-asset segment before the payoff closes the teaser. + const leadIn = openLoop + ? pool + .filter((s) => s.segment.assetId === primaryAsset && s !== hook) + .sort((a, b) => b.segment.endMs - a.segment.endMs)[0] + : undefined; + + const chosen: typeof pool = [hook]; + let total = lengthOf(hook.segment, Math.max(clipCap, Math.min(hookWindow, maxMs))); + const reserve = leadIn ? lengthOf(leadIn.segment, clipCap) : 0; + for (const candidate of pool) { + if (candidate === hook || candidate === leadIn) continue; + const length = lengthOf(candidate.segment, clipCap); + if (length < Math.min(MIN_CLIP_MS, candidate.segment.endMs - candidate.segment.startMs)) + continue; + if (total + length + reserve > maxMs) continue; + chosen.push(candidate); + total += length; + } + if (leadIn && total + reserve <= maxMs) { + chosen.push(leadIn); + total += reserve; + } + + const middle = chosen + .slice(1) + .filter((s) => s !== leadIn) + .sort((a, b) => a.score - b.score || a.segment.id.localeCompare(b.segment.id)); + const ordered = [hook, ...middle, ...(chosen.includes(leadIn!) ? [leadIn!] : [])]; + + // Lengths: start from the cap, then grow toward the minimum duration if needed. + const lengths = ordered.map((s, i) => + lengthOf(s.segment, i === 0 ? Math.max(clipCap, Math.min(hookWindow, maxMs)) : clipCap), + ); + let sum = lengths.reduce((a, b) => a + b, 0); + for (let i = 0; i < ordered.length && sum < minMs; i++) { + const full = ordered[i]!.segment.endMs - ordered[i]!.segment.startMs; + const grow = Math.min(full - lengths[i]!, minMs - sum, maxMs - sum); + if (grow > 0) { + lengths[i]! += grow; + sum += grow; + } + } + while (sum > maxMs && lengths.length > 0) { + const over = sum - maxMs; + const last = lengths.length - 1; + if (lengths[last]! - over >= MIN_CLIP_MS) { + lengths[last]! -= over; + sum -= over; + } else { + sum -= lengths.pop()!; + ordered.pop(); + } + } + + const clips = ordered.map((s, i) => { + const length = lengths[i]!; + // The lead-in ends where the payoff would begin; other clips open at their start. + const isLeadIn = s === leadIn; + const outMs = isLeadIn ? s.segment.endMs : s.segment.startMs + length; + return { + segmentId: s.segment.id, + assetId: s.segment.assetId, + inMs: outMs - length, + outMs, + }; + }); + + notes.push( + `hook: segment ${hook.segment.id} opens inside the ${hookWindow} ms hook window`, + ...middle.map((s) => `build: segment ${s.segment.id} (score ${s.score.toFixed(2)})`), + ); + if (leadIn && ordered.includes(leadIn)) { + notes.push(`open loop: segment ${leadIn.segment.id} cuts before payoff ${payoff.segment.id}`); + } else if (openLoop) { + notes.push(`open loop: payoff ${payoff.segment.id} is left out`); + } + if (input.segments.some((s) => s.paidOnly)) notes.push("paid-only segments excluded"); + if (sum < minMs) notes.push(`only ${sum} ms of usable material; profile minimum is ${minMs} ms`); + return { clips, hookSegmentId: hook.segment.id, notes }; } -/** Regularized logistic regression of conversion on features; null below TEASER_LEARNING_MIN_SAMPLES. */ +const LEARNING_RATE = 0.5; +const LEARNING_ITERATIONS = 400; +const L2 = 0.05; +const SHRINKAGE = 50; + +/** + * Regularized logistic regression of conversion on features; null below + * TEASER_LEARNING_MIN_SAMPLES samples with views. + * + * Each sample contributes `conversions` successes out of `views` trials, + * weighted so every publication counts equally. Full-batch gradient descent + * from zero with a fixed iteration count keeps it deterministic. Positive + * coefficients of features that vary in the data redistribute those features' + * prior mass; features without variation keep their prior. The result is + * shrunk toward the prior by n / (n + 50) so a small sample cannot overturn it. + */ export function fitTeaserWeights( - _samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>, - _prior: TeaserWeights, + samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>, + prior: TeaserWeights, ): TeaserWeights | null { - return notImplemented("fitTeaserWeights"); + const usable = samples.filter((s) => s.views > 0 && s.conversions >= 0); + if (usable.length < TEASER_LEARNING_MIN_SAMPLES) return null; + const k = TEASER_FEATURES.length; + const xs = usable.map((s) => TEASER_FEATURES.map((f) => clamp01(s.features[f] ?? 0))); + const ys = usable.map((s) => Math.min(1, s.conversions / s.views)); + const beta = new Array(k).fill(0); + let intercept = 0; + for (let iteration = 0; iteration < LEARNING_ITERATIONS; iteration++) { + const gradient = new Array(k).fill(0); + let gradientIntercept = 0; + for (let i = 0; i < xs.length; i++) { + const x = xs[i]!; + let z = intercept; + for (let j = 0; j < k; j++) z += beta[j]! * x[j]!; + const error = 1 / (1 + Math.exp(-z)) - ys[i]!; + gradientIntercept += error; + for (let j = 0; j < k; j++) gradient[j]! += error * x[j]!; + } + intercept -= (LEARNING_RATE * gradientIntercept) / xs.length; + for (let j = 0; j < k; j++) { + beta[j]! -= LEARNING_RATE * (gradient[j]! / xs.length + L2 * beta[j]!); + } + } + // Only features that vary in the data are re-weighted; they share their prior mass. + const varies = TEASER_FEATURES.map((_, j) => xs.some((x) => x[j] !== xs[0]![j])); + const mass = TEASER_FEATURES.reduce( + (sum, f, j) => sum + (varies[j] ? Math.max(0, prior[f] ?? 0) : 0), + 0, + ); + const positive = beta.map((b, j) => (varies[j] ? Math.max(0, b) : 0)); + const positiveTotal = positive.reduce((a, b) => a + b, 0); + const alpha = usable.length / (usable.length + SHRINKAGE); + return Object.fromEntries( + TEASER_FEATURES.map((feature, j) => { + const base = prior[feature] ?? 0; + const learned = varies[j] && positiveTotal > 0 ? (positive[j]! / positiveTotal) * mass : base; + const value = (1 - alpha) * base + alpha * learned; + return [feature, Math.round(value * 1e6) / 1e6]; + }), + ) as TeaserWeights; } -/** Deterministic first-cut EDL for a project and profile; the agent refines it. */ -export function autoPlanEdl(_input: { +const CTA_CARD_MS = 2500; + +function pickTarget(profile: PlatformProfile, goal: MediaProjectGoal, availableMs: number): number { + const { hardMinMs, hardMaxMs, recommended } = profile.duration; + const options = [...recommended] + .filter((r) => r >= hardMinMs && r <= hardMaxMs) + .sort((a, b) => a - b); + let target: number; + if (goal === "full_release" || goal === "custom") { + target = Math.min(hardMaxMs, availableMs); + } else if (options.length > 0) { + target = options[Math.floor((options.length - 1) / 2)]!; + } else { + target = Math.min(hardMaxMs, 30_000); + } + return Math.max(hardMinMs, Math.min(hardMaxMs, target)); +} + +/** + * Deterministic first-cut EDL for a project and profile; the agent refines it. + * + * Teaser-style goals use selectTeaserSegments; full releases and custom + * projects take segments in order (paid-only ones only for non-social + * profiles). Reframing is "auto" whenever the source aspect differs from the + * output, captions follow the profile when the material has speech, the + * brand intro only plays where introFits allows it, and a teaser ends on the + * brand end card (or a CTA text card). Person and mark defaults become + * redactions, and the AI-alteration metadata follows from them. + */ +export function autoPlanEdl(input: { goal: MediaProjectGoal; profile: PlatformProfile; brandKit: BrandKit | null; @@ -81,18 +437,223 @@ export function autoPlanEdl(_input: { marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; ctaText: string | null; }): Edl { - return notImplemented("autoPlanEdl"); + const { goal, profile, brandKit: kit } = input; + const W = profile.video.width; + const H = profile.video.height; + const segments = input.segments.filter((s) => { + const asset = input.assets.get(s.assetId); + return ( + asset && s.endMs > s.startMs && (asset.durationMs === null || s.endMs <= asset.durationMs) + ); + }); + const available = segments.reduce((sum, s) => sum + (s.endMs - s.startMs), 0); + const targetMs = pickTarget(profile, goal, available); + + const intro = kit?.intro && introFits(profile, kit.intro.durationMs) ? kit.intro : undefined; + let outro = goal === "teaser" ? undefined : kit?.outro; + const ctaText = input.ctaText?.trim() || null; + let ctaCard = false; + if (goal === "teaser") { + if (kit?.endCard) { + outro = { + ...kit.endCard, + params: { ...kit.endCard.params, ...(ctaText ? { cta: ctaText.slice(0, 200) } : {}) }, + }; + } else if (ctaText) { + ctaCard = true; + } + } + const bookendsMs = (intro?.durationMs ?? 0) + (outro?.durationMs ?? 0); + const budget = Math.max(MIN_CLIP_MS, targetMs - bookendsMs); + + let picks: Array<{ segmentId: string; assetId: string; inMs: number; outMs: number }>; + if (goal === "full_release" || goal === "custom") { + const allowPaid = profile.kind !== "free_social"; + const ordered = segments + .filter((s) => allowPaid || !s.paidOnly) + .sort( + (a, b) => + (a.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) - + (b.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) || + a.assetId.localeCompare(b.assetId) || + a.startMs - b.startMs, + ); + picks = []; + let used = 0; + for (const s of ordered) { + const length = Math.min(s.endMs - s.startMs, budget - used); + if (length < MIN_CLIP_MS) break; + picks.push({ + segmentId: s.id, + assetId: s.assetId, + inMs: s.startMs, + outMs: s.startMs + length, + }); + used += length; + } + } else { + picks = selectTeaserSegments({ + segments, + profile, + goal, + targetMs: budget, + weights: input.weights, + endCardMs: 0, + }).clips; + } + if (picks.length === 0) throw new Error("no usable segments for this plan"); + + let atMs = 0; + const lut = Boolean(kit?.lutArtifactId); + const video = picks.map((pick, index) => { + const asset = input.assets.get(pick.assetId)!; + const sameAspect = + !asset.width || !asset.height || Math.abs(asset.width / asset.height / (W / H) - 1) <= 0.01; + const clip = { + id: `c${index + 1}`, + assetId: pick.assetId, + inMs: pick.inMs, + outMs: pick.outMs, + atMs, + speed: 1, + reframe: sameAspect ? ("none" as const) : ("auto" as const), + ...(lut ? { color: { useBrandLut: true } } : {}), + segmentId: pick.segmentId, + }; + atMs += pick.outMs - pick.inMs; + return clip; + }); + const mainMs = atMs; + const introLength = intro?.durationMs ?? 0; + const totalMs = introLength + mainMs + (outro?.durationMs ?? 0); + + const usedAssets = new Set(picks.map((p) => p.assetId)); + const hasSpeech = [...usedAssets].some((id) => input.assets.get(id)?.hasSpeech); + const preset = kit?.captionPresets[0]; + const captions = + (profile.captions.recommended || profile.captions.required) && hasSpeech + ? { + mode: profile.export.subtitles ? ("both" as const) : ("burned" as const), + ...(preset ? { presetId: preset.id } : {}), + maxCharsPerLine: W < H ? 24 : 32, + wordHighlight: preset?.template === "karaoke", + position: "lower" as const, + } + : undefined; + + const zones = profile.safeZones; + const overlays: Array> = []; + if (kit?.watermark) { + overlays.push({ id: "watermark", kind: "watermark", atMs: 0, durationMs: totalMs }); + } + if (ctaCard && ctaText) { + const cardMs = Math.min(CTA_CARD_MS, mainMs); + const left = zones.leftPct / 100 + 0.02; + const right = zones.rightPct / 100 + 0.02; + overlays.push({ + id: "cta", + kind: "cta_end_card", + atMs: introLength + mainMs - cardMs, + durationMs: cardMs, + text: ctaText.slice(0, 200), + box: { x: left, y: 0.42, w: Math.max(0.1, 1 - left - right), h: 0.12 }, + }); + } + + const protectedPeople = input.people.flatMap((person) => { + const mode = AnonymizeModeSchema.safeParse(person.anonymizeByDefault); + return mode.success && mode.data !== "none" ? [{ id: person.id, mode: mode.data }] : []; + }); + const faces: Array<{ + id: string; + target: { personId: string } | { unassignedFaces: true }; + mode: Exclude; + }> = protectedPeople.map((person, index) => ({ + id: `face-${index + 1}`, + target: { personId: person.id }, + mode: person.mode, + })); + if (faces.length > 0) { + // Unknown faces next to a protected person are blurred too; the gate blocks them otherwise. + faces.push({ id: "face-unassigned", target: { unassignedFaces: true }, mode: "blur" }); + } + const peopleIds = new Set(input.people.map((p) => p.id)); + const marks = input.marks.flatMap((mark) => { + if (mark.treatment === "none") return []; + const relevant = + (mark.assetId !== null && usedAssets.has(mark.assetId)) || + (mark.personId !== null && peopleIds.has(mark.personId)) || + (mark.assetId === null && mark.personId === null); + if (!relevant) return []; + const treatment = mark.treatment as "inpaint" | "blur" | "cover" | "reframe_out"; + return [{ markId: mark.id, treatment }]; + }); + + const aiAlterations: AiAlteration[] = []; + if (faces.some((f) => f.mode === "synthetic_identity")) aiAlterations.push("synthetic_identity"); + if (marks.some((m) => m.treatment === "inpaint")) aiAlterations.push("inpaint"); + + return EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: profile.id, + width: W, + height: H, + fps: profile.video.fps, + durationTargetMs: targetMs, + durationHardMinMs: profile.duration.hardMinMs, + durationHardMaxMs: profile.duration.hardMaxMs, + }, + tracks: { video, audio: [] }, + transitions: [], + ...(captions ? { captions } : {}), + overlays, + ...(intro ? { intro } : {}), + ...(outro ? { outro } : {}), + redactions: { faces, marks, regions: [] }, + metadata: { tags: [], aiAltered: aiAlterations.length > 0, aiAlterations }, + }); } +const THUMBNAIL_WEIGHTS = { sharpness: 0.4, face: 0.25, thirds: 0.2, contrast: 0.15 }; + +/** + * Best thumbnail frame times, spread out: candidates are ranked by sharpness, + * face presence, rule of thirds and contrast, and each pick must be at least + * a tenth of the candidate span (min 1 s) away from earlier picks while + * enough candidates remain. + */ export function rankThumbnailFrames( - _candidates: Array<{ + candidates: Array<{ tMs: number; sharpness: number; face: number; thirds: number; contrast: number; }>, - _count: number, + count: number, ): number[] { - return notImplemented("rankThumbnailFrames"); + if (count <= 0 || candidates.length === 0) return []; + const ranked = candidates + .map((c) => ({ + tMs: c.tMs, + score: + THUMBNAIL_WEIGHTS.sharpness * clamp01(c.sharpness) + + THUMBNAIL_WEIGHTS.face * clamp01(c.face) + + THUMBNAIL_WEIGHTS.thirds * clamp01(c.thirds) + + THUMBNAIL_WEIGHTS.contrast * clamp01(c.contrast), + })) + .sort((a, b) => b.score - a.score || a.tMs - b.tMs); + const times = candidates.map((c) => c.tMs); + const spacing = Math.max(1000, (Math.max(...times) - Math.min(...times)) / 10); + const picked: number[] = []; + for (const candidate of ranked) { + if (picked.length >= count) break; + if (picked.every((t) => Math.abs(t - candidate.tMs) >= spacing)) picked.push(candidate.tMs); + } + for (const candidate of ranked) { + if (picked.length >= count) break; + if (!picked.includes(candidate.tMs)) picked.push(candidate.tMs); + } + return picked; } From a6598f09d930e00690e4fc6f1757fb2c450e5470 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:06:02 +0000 Subject: [PATCH 13/16] test(media): cover clustering, EDL, compiler, teaser, gate and marketing Includes compiler snapshots, injection-proof escaping checks, and an ffmpeg integration test on synthetic testsrc/sine inputs that is skipped when ffmpeg is not installed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- .../src/__snapshots__/compile.test.ts.snap | 491 +++++++++++++++ packages/media/src/clustering.test.ts | 206 +++++++ packages/media/src/compile.ffmpeg.test.ts | 322 ++++++++++ packages/media/src/compile.test.ts | 576 ++++++++++++++++++ packages/media/src/edl.test.ts | 337 ++++++++++ packages/media/src/gate.test.ts | 531 ++++++++++++++++ packages/media/src/marketing.test.ts | 320 ++++++++++ packages/media/src/teaser.test.ts | 334 ++++++++++ packages/media/src/test-fixtures.ts | 135 ++++ 9 files changed, 3252 insertions(+) create mode 100644 packages/media/src/__snapshots__/compile.test.ts.snap create mode 100644 packages/media/src/clustering.test.ts create mode 100644 packages/media/src/compile.ffmpeg.test.ts create mode 100644 packages/media/src/compile.test.ts create mode 100644 packages/media/src/edl.test.ts create mode 100644 packages/media/src/gate.test.ts create mode 100644 packages/media/src/marketing.test.ts create mode 100644 packages/media/src/teaser.test.ts create mode 100644 packages/media/src/test-fixtures.ts diff --git a/packages/media/src/__snapshots__/compile.test.ts.snap b/packages/media/src/__snapshots__/compile.test.ts.snap new file mode 100644 index 0000000..94005f5 --- /dev/null +++ b/packages/media/src/__snapshots__/compile.test.ts.snap @@ -0,0 +1,491 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`compileRenderPlan > compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "subtitles_srt", + "path": "/work/render-1/captions.srt", + }, + { + "kind": "subtitles_vtt", + "path": "/work/render-1/captions.vtt", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "final", + "steps": [ + { + "content": "[Script Info] +ScriptType: v4.00+ +PlayResX: 1080 +PlayResY: 1920 +WrapStyle: 2 +ScaledBorderAndShadow: yes + +[V4+ Styles] +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding +Style: Default,Example Sans,96,&H00FFFFFF,&H00FFFFFF,&H00000000,&H80000000,0,0,0,0,100,100,0,0,1,2,0,2,92,168,422,1 + +[Events] +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text +Dialogue: 0,0:00:00.20,0:00:01.80,Default,,0,0,0,,Wait for this! +Dialogue: 0,0:00:02.10,0:00:03.40,Default,,0,0,0,,Here it is --> now. +", + "id": "captions-ass", + "kind": "write_file", + "path": "/work/render-1/captions.ass", + }, + { + "content": "1 +00:00:00,200 --> 00:00:01,800 +Wait for this! + +2 +00:00:02,100 --> 00:00:03,400 +Here it is -> now. +", + "id": "captions-srt", + "kind": "write_file", + "path": "/work/render-1/captions.srt", + }, + { + "content": "WEBVTT + +00:00:00.200 --> 00:00:01.800 +Wait for this! + +00:00:02.100 --> 00:00:03.400 +Here it <b>is</b> --> now. +", + "id": "captions-vtt", + "kind": "write_file", + "path": "/work/render-1/captions.vtt", + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "5", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-i", + "/brand/watermark.png", + "-filter_complex", + "[0:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,170.82\\,if(lt(t\\,1.5)\\,170.82+(167.81)*(t-0)/1.5\\,if(lt(t\\,2)\\,338.62+(19.97)*(t-1.5)/0.5\\,358.59))):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,1.5)\\,0+(0)*(t-0)/1.5\\,if(lt(t\\,2)\\,0+(0)*(t-1.5)/0.5\\,0))),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,1232\\,if(lt(t\\,2)\\,1232+(0)*(t-0)/2\\,1232)):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,2)\\,0+(0)*(t-0)/2\\,0)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c2];[v_c1][a_c1][v_c2][a_c2]concat=n=2:v=1:a=1[cv1][ca2];[2:v]format=rgba,scale=162:-2,colorchannelmixer=aa=0.6[img3];[cv1][img3]overlay=x=main_w-overlay_w-152:y=214:enable=between(t\\,0\\,4)[ov4];[ov4]ass=filename=/work/render-1/captions.ass:fontsdir=/brand/fonts[cap5];[cap5]format=yuv420p[vout];[ca2]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "20", + "-profile:v", + "high", + "-maxrate", + "8000k", + "-bufsize", + "16000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "192k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-metadata", + "title=Example: teaser", + "-t", + "4", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 4000, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "1.333", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; + +exports[`compileRenderPlan > compiles transitions, music ducking, voice-over and fades 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "draft", + "steps": [ + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "2", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "3", + "-t", + "1", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "4", + "-t", + "1", + "-i", + "/media/asset-b/original.mov", + "-ss", + "0", + "-t", + "8", + "-i", + "/brand/music/bed.wav", + "-ss", + "0", + "-t", + "2", + "-i", + "/work/render-1/voiceover.wav", + "-filter_complex", + "[0:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/0.5,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=4,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,atempo=0.5,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=4,asetpts=PTS-STARTPTS[a_c2];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c3];[2:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c3];[3:v]crop=w=608:h=1080:x=656:y=0,scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c4];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c4];[v_c1][v_c2]xfade=transition=fade:duration=0.5:offset=1.5[xv1];[a_c1][a_c2]acrossfade=d=0.5:c1=tri:c2=tri[xa2];color=c=black:s=1080x1920:r=30,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=0.5,setpts=PTS-STARTPTS[gv3];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=0.5,asetpts=PTS-STARTPTS[ga4];[xv1][xa2][gv3][ga4][v_c3][a_c3][v_c4][a_c4]concat=n=4:v=1:a=1[cv5][ca6];[cv5]format=yuv420p[vout];[4:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,volume=-8dB,afade=t=in:st=0:d=0.5,afade=t=out:st=7:d=1[ac_bed];[5:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,adelay=1000|1000[ac_vo];[ca6][ac_vo]amix=inputs=2:duration=first:normalize=0[speech];[speech]asplit=2[speech_mix][speech_key];[ac_bed]anull[duck_bus];[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked];[speech_mix][ducked]amix=inputs=2:duration=first:normalize=0[mixed];[mixed]loudnorm=I=-16:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "28", + "-profile:v", + "high", + "-maxrate", + "3000k", + "-bufsize", + "6000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "128k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-t", + "8", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 8000, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "2.667", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; + +exports[`compileRenderPlan > compiles voice disguise, metadata stripping, intro/outro and text overlays 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "final", + "steps": [ + { + "fps": 30, + "height": 1920, + "id": "motion-1", + "kind": "motion", + "outPath": "/work/render-1/motion-1.mov", + "template": { + "durationMs": 1000, + "params": {}, + "templateId": "outro-simple", + }, + "width": 1080, + }, + { + "fps": 30, + "height": 1920, + "id": "motion-2", + "kind": "motion", + "outPath": "/work/render-1/motion-2.mov", + "template": { + "durationMs": 1000, + "params": {}, + "templateId": "lower-third-a", + }, + "width": 1080, + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "3", + "-i", + "/media/asset-a/proxy.mp4", + "-i", + "/work/render-1/intro.mov", + "-i", + "/work/render-1/motion-1.mov", + "-i", + "/work/render-1/motion-2.mov", + "-filter_complex", + "[0:v]deshake,crop=w=406:h=720:x=if(lt(t\\,0)\\,373\\,if(lt(t\\,3)\\,373+(768)*(t-0)/3\\,1141)):y=if(lt(t\\,0)\\,180\\,if(lt(t\\,3)\\,180+(0)*(t-0)/3\\,180)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,eq=brightness=0.05:contrast=1:saturation=1.1,colorbalance=rm=0.02:bm=-0.02,lut3d=file=/brand/look\\\\\\'s.cube,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=3,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[a_c1];[a_c1]rubberband=pitch=0.793701:formant=preserved,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[vd1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1.5,setpts=PTS-STARTPTS[intro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1.5,asetpts=PTS-STARTPTS[intro_a];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[outro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[outro_a];[intro_v][intro_a][v_c1][vd1][outro_v][outro_a]concat=n=3:v=1:a=1[prog_v][prog_a];[3:v]format=yuva420p,scale=1080:1920,setpts=PTS-STARTPTS+0/TB[mo2];[prog_v][mo2]overlay=x=0:y=0:eof_action=pass:enable=between(t\\,0\\,1)[ov3];color=c=0xFFAA00:s=1080x10:r=30:d=5.5[pb4];[ov3][pb4]overlay=x=-overlay_w+main_w*min(1\\,(t-1.5)/3):y=1504:enable=between(t\\,1.5\\,4.5)[ov5];[ov5]drawtext=fontfile=/brand/fonts/ExampleSans.ttf:text=Ana\\\\\\'s\\\\: \\[studio\\]\\; 100\\\\\\\\% real:fontsize=77:fontcolor=0xFFFFFF:x=(w-text_w)/2:y=h-text_h-560:box=1:boxcolor=0x112233@0.85:boxborderw=31:enable=between(t\\,2\\,3.5)[ov6];[ov6]format=yuv420p[vout];[prog_a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "20", + "-profile:v", + "high", + "-maxrate", + "8000k", + "-bufsize", + "16000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "192k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-t", + "5.5", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 5500, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "2.5", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; diff --git a/packages/media/src/clustering.test.ts b/packages/media/src/clustering.test.ts new file mode 100644 index 0000000..c2e333d --- /dev/null +++ b/packages/media/src/clustering.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from "vitest"; +import type { ClusterAsset } from "./clustering.js"; +import { + buildSessions, + calendarDay, + clusterFaces, + clusterThemes, + ruleLabel, +} from "./clustering.js"; + +const at = (iso: string) => new Date(iso); + +function asset( + id: string, + captured: string | null, + extra: Partial = {}, +): ClusterAsset { + return { + id, + kind: "photo", + capturedAt: captured ? at(captured) : null, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + embedding: null, + ...extra, + }; +} + +const vec = (...values: number[]) => new Float32Array(values); + +describe("buildSessions", () => { + it("splits on gaps longer than 45 minutes and sorts by capture time", () => { + const sessions = buildSessions([ + asset("c", "2026-05-01T10:40:00Z"), + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:20:00Z"), + asset("d", "2026-05-01T11:26:00Z"), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]); + expect(sessions[0]!.key).toBe("session:a"); + expect(sessions[0]!.startAt).toEqual(at("2026-05-01T10:00:00Z")); + expect(sessions[0]!.endAt).toEqual(at("2026-05-01T10:40:00Z")); + }); + + it("keeps a gap of exactly 45 minutes in one session", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:45:00Z"), + ]); + expect(sessions).toHaveLength(1); + }); + + it("puts undated assets into one trailing session", () => { + const sessions = buildSessions([ + asset("z", null), + asset("a", "2026-05-01T10:00:00Z"), + asset("m", null), + asset("bad", null, { capturedAt: new Date(Number.NaN) }), + ]); + expect(sessions.map((s) => s.key)).toEqual(["session:a", "session:undated"]); + expect(sessions[1]!.assetIds).toEqual(["bad", "m", "z"]); + expect(sessions[1]!.startAt).toBeNull(); + }); + + it("ignores duplicate ids and mixes photos and videos", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z"), + asset("v", "2026-05-01T10:05:00Z", { kind: "video" }), + asset("a", "2026-05-01T23:00:00Z"), + asset("l", "2026-05-01T10:06:00Z", { kind: "live_photo" }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "v", "l"]]); + }); + + it("splits on a location jump over 2 km and reports a rounded centroid", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z", { gpsLat: 10.0, gpsLon: 20.0 }), + asset("b", "2026-05-01T10:05:00Z"), + asset("c", "2026-05-01T10:10:00Z", { gpsLat: 10.004, gpsLon: 20.004 }), + asset("d", "2026-05-01T10:15:00Z", { gpsLat: 10.1, gpsLon: 20.0 }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]); + expect(sessions[0]!.centroid).toEqual({ lat: 10, lon: 20 }); + expect(sessions[1]!.centroid).toEqual({ lat: 10.1, lon: 20 }); + }); + + it("breaks on a calendar-day change only with a gap over 3 hours, in the given time zone", () => { + const assets = [asset("a", "2026-05-01T22:30:00Z"), asset("b", "2026-05-02T02:00:00Z")]; + // Long gap rule disabled so only the day-break rule decides. + const options = { gapMinutes: 24 * 60 }; + expect(buildSessions(assets, options)).toHaveLength(2); + // In UTC-10 both are on 1 May: one session. + expect(buildSessions(assets, { ...options, timeZone: "Pacific/Honolulu" })).toHaveLength(1); + // Crossing midnight with a short gap stays one session. + expect( + buildSessions( + [asset("x", "2026-05-01T23:50:00Z"), asset("y", "2026-05-02T00:20:00Z")], + options, + ), + ).toHaveLength(1); + expect(calendarDay(at("2026-05-01T22:30:00Z"), "Europe/Berlin")).toBe("2026-05-02"); + }); + + it("merges neighbours with similar embeddings and a gap under 2 hours", () => { + const red = vec(1, 0, 0); + const blue = vec(0, 1, 0); + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z", { embedding: red }), + asset("b", "2026-05-01T11:00:00Z", { embedding: vec(0.95, 0.05, 0) }), + asset("c", "2026-05-01T12:00:00Z", { embedding: blue }), + asset("d", "2026-05-01T15:00:00Z", { embedding: blue }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b"], ["c"], ["d"]]); + }); + + it("is deterministic for any input order", () => { + const assets = [ + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:00:00Z"), + asset("c", "2026-05-02T10:00:00Z"), + asset("d", null), + ]; + const forward = buildSessions(assets); + const reversed = buildSessions([...assets].reverse()); + expect(reversed).toEqual(forward); + expect(forward[0]!.assetIds).toEqual(["a", "b"]); + }); +}); + +describe("clusterThemes and clusterFaces", () => { + const items = [ + { id: "p3", vector: vec(0.9, 0.1, 0) }, + { id: "p1", vector: vec(1, 0, 0) }, + { id: "q1", vector: vec(0, 1, 0) }, + { id: "p2", vector: vec(0.95, 0.05, 0) }, + { id: "q2", vector: vec(0.05, 0.95, 0) }, + { id: "r1", vector: vec(0, 0, 1) }, + ]; + + it("groups by average linkage with stable keys, independent of input order", () => { + const themes = clusterThemes(items); + expect(themes).toEqual([ + { key: "theme:p1", ids: ["p1", "p2", "p3"] }, + { key: "theme:q1", ids: ["q1", "q2"] }, + { key: "theme:r1", ids: ["r1"] }, + ]); + expect(clusterThemes([...items].reverse())).toEqual(themes); + expect(clusterThemes(items, { minSize: 2 }).map((t) => t.key)).toEqual([ + "theme:p1", + "theme:q1", + ]); + expect(clusterThemes(items, { similarity: -1 })).toHaveLength(1); + expect(clusterThemes([])).toEqual([]); + }); + + it("uses average rather than single linkage", () => { + // b is close to a and c, but a and c are far apart: average linkage keeps c out. + const chain = [ + { id: "a", vector: vec(1, 0) }, + { id: "b", vector: vec(Math.cos(0.4), Math.sin(0.4)) }, + { id: "c", vector: vec(Math.cos(1.0), Math.sin(1.0)) }, + ]; + expect(clusterThemes(chain, { similarity: 0.8 }).map((g) => g.ids)).toEqual([ + ["a", "b"], + ["c"], + ]); + }); + + it("returns face groups with a mean centroid", () => { + const faces = clusterFaces([ + { id: "t1", vector: vec(1, 0) }, + { id: "t2", vector: vec(0.8, 0.2) }, + { id: "t3", vector: vec(0, 1) }, + ]); + expect(faces.map((f) => f.key)).toEqual(["person:t1", "person:t3"]); + expect(Array.from(faces[0]!.centroid)).toEqual([0.8999999761581421, 0.10000000149011612]); + expect(() => + clusterFaces([ + { id: "x", vector: vec(1) }, + { id: "y", vector: vec(1, 2) }, + ]), + ).toThrow(); + }); +}); + +describe("ruleLabel", () => { + it("combines the dominant album with a locale-aware date range", () => { + const label = ruleLabel( + { + startAt: at("2026-05-01T10:00:00Z"), + endAt: at("2026-05-03T10:00:00Z"), + albumNames: ["Trip", "Favorites", "Trip"], + }, + "en-US", + ); + expect(label.startsWith("Trip · ")).toBe(true); + expect(label).toMatch(/May 1\D+3, 2026/); + expect( + ruleLabel({ startAt: at("2026-05-01T10:00:00Z"), endAt: null, albumNames: [] }, "de-DE"), + ).toBe("01.05.2026"); + expect(ruleLabel({ startAt: null, endAt: null, albumNames: [" "] })).toBe(""); + expect(ruleLabel({ startAt: null, endAt: null, albumNames: ["B", "A"] })).toBe("A"); + }); +}); diff --git a/packages/media/src/compile.ffmpeg.test.ts b/packages/media/src/compile.ffmpeg.test.ts new file mode 100644 index 0000000..cd6051e --- /dev/null +++ b/packages/media/src/compile.ffmpeg.test.ts @@ -0,0 +1,322 @@ +import { execFile, execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import type { RenderPlan } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { CompileInput } from "./compile.js"; +import { compileRenderPlan, escapeFilterText, escapeFilterValue } from "./compile.js"; +import { edlDurationMs } from "./edl.js"; +import { findMetadataLeaks } from "./gate.js"; +import { testEdl, testKit, testProfile } from "./test-fixtures.js"; + +const run = promisify(execFile); + +function onPath(binary: string): boolean { + try { + execFileSync(binary, ["-version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +const hasFfmpeg = onPath("ffmpeg") && onPath("ffprobe"); +const hasRubberband = + hasFfmpeg && + execFileSync("ffmpeg", ["-hide_banner", "-filters"]).toString().includes(" rubberband "); + +async function ffmpeg(args: string[]): Promise { + await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]); +} + +async function execute(plan: RenderPlan): Promise { + for (const step of plan.steps) { + if (step.kind === "write_file") await writeFile(step.path, step.content); + else if (step.kind === "ffmpeg") await run("ffmpeg", ["-loglevel", "error", ...step.argv]); + else throw new Error(`unexpected ${step.kind} step`); + } +} + +interface Probe { + durationMs: number; + width: number; + height: number; + tags: Record; +} + +async function probe(file: string): Promise { + const { stdout } = await run("ffprobe", [ + "-v", + "error", + "-show_format", + "-show_streams", + "-of", + "json", + file, + ]); + const json = JSON.parse(stdout) as { + format: { duration: string; tags?: Record }; + streams: Array<{ + codec_type: string; + width?: number; + height?: number; + tags?: Record; + }>; + }; + const tags: Record = {}; + for (const [key, value] of Object.entries(json.format.tags ?? {})) tags[`format.${key}`] = value; + json.streams.forEach((stream, index) => { + for (const [key, value] of Object.entries(stream.tags ?? {})) + tags[`stream.${index}.${key}`] = value; + }); + const video = json.streams.find((s) => s.codec_type === "video"); + return { + durationMs: Number(json.format.duration) * 1000, + width: video?.width ?? 0, + height: video?.height ?? 0, + tags, + }; +} + +describe.skipIf(!hasFfmpeg)("compiled plans run on ffmpeg", () => { + let dir: string; + let source: string; + let watermark: string; + let music: string; + let intro: string; + let lut: string; + + beforeAll(async () => { + dir = await mkdtemp(path.join(tmpdir(), "media-compile-")); + source = path.join(dir, "source.mp4"); + watermark = path.join(dir, "watermark.png"); + music = path.join(dir, "music.wav"); + intro = path.join(dir, "intro.mov"); + lut = path.join(dir, "brand look.cube"); + // Identity 2x2x2 LUT. + const lattice = []; + for (const b of [0, 1]) + for (const g of [0, 1]) for (const r of [0, 1]) lattice.push(`${r} ${g} ${b}`); + await writeFile(lut, `LUT_3D_SIZE 2\n${lattice.join("\n")}\n`); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc2=size=480x270:rate=30:duration=8", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=8", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + "-shortest", + // Tags a phone would write; the export must drop every one of them. + "-metadata", + "creation_time=2020-01-01T10:00:00Z", + "-metadata", + "location=+00.0000+000.0000/", + "-metadata", + "make=ExampleCam", + source, + ]); + await ffmpeg(["-f", "lavfi", "-i", "color=c=white:s=64x32", "-frames:v", "1", watermark]); + await ffmpeg(["-f", "lavfi", "-i", "sine=frequency=220:duration=6", music]); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc=size=320x180:rate=30:duration=1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + intro, + ]); + }); + + afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }); + }); + + const baseInput = (overrides: Partial): CompileInput => ({ + edl: testEdl(), + profile: testProfile(), + brandKit: testKit(), + quality: "draft", + workDir: dir, + sources: new Map([ + [ + "asset-a", + { + path: source, + width: 480, + height: 270, + durationMs: 8000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + ]), + transcripts: new Map([ + [ + "asset-a", + [ + { startMs: 200, endMs: 600, text: "Wait" }, + { startMs: 650, endMs: 1200, text: "for" }, + { startMs: 1250, endMs: 1800, text: "this!" }, + { startMs: 5100, endMs: 5600, text: "Here" }, + { startMs: 5650, endMs: 6400, text: "it is." }, + ], + ], + ]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.3, cy: 0.5, strength: 1 }, + { tMs: 6000, cx: 0.7, cy: 0.5, strength: 1 }, + ], + ], + ]), + motionClips: new Map(), + files: { + lutPath: null, + watermarkPath: watermark, + fonts: new Map(), + audio: new Map([["music-1", music]]), + }, + videoEncoder: "libx264", + ...overrides, + }); + + it("renders 16:9 to 9:16 with auto-reframe, captions, watermark, ducked music and clean metadata", async () => { + const edl = testEdl({ + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 1500, reframe: "auto" }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: "music-1", + inMs: 0, + outMs: 3500, + atMs: 0, + gainDb: -6, + fadeOutMs: 500, + duckUnderSpeech: true, + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }], + captions: { mode: "both" }, + overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 3500 }], + metadata: { title: "Example teaser" }, + }); + const plan = compileRenderPlan(baseInput({ edl })); + await execute(plan); + const video = plan.outputs.find((o) => o.kind === "video")!.path; + const result = await probe(video); + expect(result.width).toBe(1080); + expect(result.height).toBe(1920); + expect(Math.abs(result.durationMs - edlDurationMs(edl))).toBeLessThanOrEqual(1000 / 30 + 1); + const keys = Object.keys(result.tags).map((k) => k.toLowerCase()); + expect(keys.some((k) => k.includes("creation_time") || k.includes("location"))).toBe(false); + expect(findMetadataLeaks(result.tags)).toEqual([]); + expect(result.tags["format.title"]).toBe("Example teaser"); + + const srt = await readFile(plan.outputs.find((o) => o.kind === "subtitles_srt")!.path, "utf8"); + expect(srt).toContain("Wait for this!"); + const poster = plan.outputs.find((o) => o.kind === "poster")!.path; + expect(existsSync(poster)).toBe(true); + expect((await probe(poster)).tags).toEqual({}); + }); + + it("renders an intro, voice disguise and an escaped text overlay", async () => { + const edl = testEdl({ + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 1000, + outMs: 3000, + atMs: 0, + reframe: "none", + stabilize: true, + color: { exposure: 0.3, temperature: -0.4, saturation: 1.2, useBrandLut: true }, + }, + ], + }, + intro: { templateId: "intro-minimal", durationMs: 1000 }, + overlays: [ + { + id: "t1", + kind: "text", + atMs: 1000, + durationMs: 1500, + text: "50% off: it's [0:v]; rm -rf / %{pts} \\ done", + box: { x: 0.1, y: 0.3, w: 0.8, h: 0.05 }, + }, + { id: "lt", kind: "lower_third", atMs: 1200, durationMs: 1000, text: "Lower third" }, + { id: "pb", kind: "progress_bar", atMs: 1000, durationMs: 2000 }, + ], + redactions: { voice: { method: hasRubberband ? "pitch" : "formant", semitones: 3 } }, + metadata: {}, + }); + const plan = compileRenderPlan( + baseInput({ + edl, + motionClips: new Map([["intro", intro]]), + quality: "final", + files: { ...baseInput({}).files, lutPath: lut }, + }), + ); + await execute(plan); + const result = await probe(plan.outputs[0]!.path); + expect(result.width).toBe(1080); + expect(Math.abs(result.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1); + expect(findMetadataLeaks(result.tags)).toEqual([]); + expect(Object.keys(result.tags).some((k) => k.endsWith("title"))).toBe(false); + }); + + it("draws escaped text exactly like the literal text", async () => { + const text = "it's 50% [0:v]; a,b=c \\ %{pts} 'q'"; + const literal = path.join(dir, "literal.txt"); + await writeFile(literal, text); + const frame = async (filter: string) => { + const { stdout } = await run("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `color=c=black:s=320x80:d=0.1,${filter}`, + "-frames:v", + "1", + "-f", + "framemd5", + "-", + ]); + return stdout.split("\n").filter((line) => line && !line.startsWith("#")); + }; + const escaped = await frame(`drawtext=text=${escapeFilterText(text)}:fontcolor=white:x=4:y=20`); + const reference = await frame( + `drawtext=expansion=none:textfile=${escapeFilterValue(literal)}:fontcolor=white:x=4:y=20`, + ); + const blank = await frame("null"); + expect(escaped).toEqual(reference); + expect(escaped).not.toEqual(blank); + }); +}); diff --git a/packages/media/src/compile.test.ts b/packages/media/src/compile.test.ts new file mode 100644 index 0000000..1a14ebf --- /dev/null +++ b/packages/media/src/compile.test.ts @@ -0,0 +1,576 @@ +import type { RenderPlan } from "@rakazo/contracts"; +import { RenderPlanSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { CaptionWord, CompileInput } from "./compile.js"; +import { + atempoChain, + autoReframe, + buildCaptionCues, + compileRenderPlan, + escapeFilterText, + escapeFilterValue, + outputCaptionWords, + reframeExcluding, + toAss, + toSrt, + toVtt, +} from "./compile.js"; +import { testEdl, testKit, testProfile } from "./test-fixtures.js"; + +const words: CaptionWord[] = [ + { startMs: 200, endMs: 600, text: "Wait" }, + { startMs: 650, endMs: 1200, text: "for" }, + { startMs: 1250, endMs: 1800, text: "this!" }, + { startMs: 5100, endMs: 5600, text: "Here" }, + { startMs: 5650, endMs: 6400, text: "it is --> now." }, +]; + +function input(overrides: Partial = {}): CompileInput { + return { + edl: testEdl(), + profile: testProfile(), + brandKit: testKit(), + quality: "final", + workDir: "/work/render-1", + sources: new Map([ + [ + "asset-a", + { + path: "/media/asset-a/proxy.mp4", + width: 1920, + height: 1080, + durationMs: 10_000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + [ + "asset-b", + { + path: "/media/asset-b/original.mov", + width: 1080, + height: 1920, + durationMs: 8000, + fps: 60, + hasAudio: false, + rotation: 90, + }, + ], + ]), + transcripts: new Map([["asset-a", words]]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.2, cy: 0.5, strength: 1 }, + { tMs: 1500, cx: 0.4, cy: 0.5, strength: 1 }, + { tMs: 5500, cx: 0.8, cy: 0.4, strength: 0.5 }, + ], + ], + ]), + motionClips: new Map(), + files: { + lutPath: "/brand/look's.cube", + watermarkPath: "/brand/watermark.png", + fonts: new Map([["font-1", "/brand/fonts/ExampleSans.ttf"]]), + audio: new Map([ + ["music-1", "/brand/music/bed.wav"], + ["vo-1", "/work/render-1/voiceover.wav"], + ]), + }, + videoEncoder: "libx264", + ...overrides, + }; +} + +function ffmpegStep(plan: RenderPlan, id = "render") { + const step = plan.steps.find((s) => s.id === id); + if (step?.kind !== "ffmpeg") throw new Error(`no ffmpeg step ${id}`); + return step; +} + +function graphOf(plan: RenderPlan): string { + const argv = ffmpegStep(plan).argv; + return argv[argv.indexOf("-filter_complex") + 1]!; +} + +describe("compileRenderPlan", () => { + it("compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + captions: { mode: "both", presetId: "clean" }, + overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 }], + metadata: { title: "Example: teaser" }, + }), + }), + ); + expect(RenderPlanSchema.parse(plan)).toEqual(plan); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("crop=w=608:h=1080:x="); + expect(graph).toContain("scale=1080:1920"); + expect(graph).toContain("loudnorm=I=-14:TP=-1:LRA=11:linear=true"); + expect(graph).toContain("colorchannelmixer=aa=0.6"); + expect(graph).toContain("ass=filename=/work/render-1/captions.ass"); + expect(plan.outputs.map((o) => o.kind)).toEqual([ + "video", + "subtitles_srt", + "subtitles_vtt", + "poster", + ]); + const argv = ffmpegStep(plan).argv; + for (const flag of [ + "-map_metadata", + "-map_chapters", + "+bitexact", + "+faststart", + "yuv420p", + "bt709", + ]) { + expect(argv).toContain(flag); + } + expect(argv).toContain("title=Example: teaser"); + expect(argv.join(" ")).toContain("-c:a aac -b:a 192k -ar 48000"); + }); + + it("compiles transitions, music ducking, voice-over and fades", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" }, + { + id: "c2", + assetId: "asset-a", + inMs: 2000, + outMs: 4000, + atMs: 1500, + reframe: "none", + speed: 0.5, + }, + { + id: "c3", + assetId: "asset-a", + inMs: 3000, + outMs: 4000, + atMs: 6000, + reframe: "none", + }, + { + id: "c4", + assetId: "asset-b", + inMs: 4000, + outMs: 5000, + atMs: 7000, + reframe: "auto", + }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: "music-1", + inMs: 0, + outMs: 8000, + atMs: 0, + gainDb: -8, + fadeInMs: 500, + fadeOutMs: 1000, + duckUnderSpeech: true, + }, + { id: "vo", kind: "voiceover", artifactId: "vo-1", inMs: 0, outMs: 2000, atMs: 1000 }, + ], + }, + transitions: [ + { fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }, + { fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }, + { fromClipId: "c3", toClipId: "c4", kind: "whip", durationMs: 0 }, + ], + targetLufs: -16, + }), + quality: "draft", + }), + ); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("xfade=transition=fade:duration=0.5:offset=1.5"); + expect(graph).toContain("acrossfade=d=0.5"); + expect(graph).toContain("atempo=0.5"); + expect(graph).toContain("color=c=black:s=1080x1920"); // gap from 5.5 s to 6 s + expect(graph).toContain("sidechaincompress"); + expect(graph).toContain("adelay=1000|1000"); + expect(graph).toContain("loudnorm=I=-16"); + expect(ffmpegStep(plan).argv).toEqual(expect.arrayContaining(["veryfast", "28", "3000k"])); + }); + + it("compiles voice disguise, metadata stripping, intro/outro and text overlays", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1500 }, + outro: { templateId: "outro-simple", durationMs: 1000 }, + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 0, + outMs: 3000, + atMs: 0, + reframe: [ + { tMs: 0, cx: 0.3, cy: 0.5, scale: 1 }, + { tMs: 3000, cx: 0.7, cy: 0.5, scale: 1.5 }, + ], + stabilize: true, + color: { exposure: 0.5, temperature: 0.2, saturation: 1.1, useBrandLut: true }, + }, + ], + }, + overlays: [ + { + id: "lt", + kind: "lower_third", + atMs: 2000, + durationMs: 1500, + text: "Ana's: [studio]; 100% real", + }, + { + id: "logo-anim", + kind: "motion", + atMs: 0, + durationMs: 1000, + motion: { templateId: "lower-third-a", durationMs: 1000 }, + }, + { id: "pb", kind: "progress_bar", atMs: 1500, durationMs: 3000 }, + ], + redactions: { voice: { method: "pitch", semitones: -4 } }, + metadata: {}, + }), + motionClips: new Map([["intro", "/work/render-1/intro.mov"]]), + }), + ); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("rubberband=pitch=0.793701:formant=preserved"); + expect(graph).toContain("deshake"); + expect(graph).toContain("lut3d=file=/brand/look\\\\\\'s.cube"); + expect(graph).toContain("concat=n=3:v=1:a=1[prog_v][prog_a]"); + expect(plan.steps.filter((s) => s.kind === "motion").map((s) => s.id)).toEqual([ + "motion-1", + "motion-2", + ]); + const argv = ffmpegStep(plan).argv; + expect(argv.some((a) => a.startsWith("title="))).toBe(false); + expect(argv).toEqual(expect.arrayContaining(["-map_metadata", "-1", "-map_chapters", "-1"])); + }); + + it("uses the asetrate chain for formant disguise and HEVC settings for HEVC profiles", () => { + const plan = compileRenderPlan( + input({ + profile: testProfile({ + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "hevc", + maxBitrateKbps: 6000, + maxFileSizeMb: 100, + }, + export: { poster: false }, + }), + edl: testEdl({ redactions: { voice: { method: "formant", semitones: 12 } }, metadata: {} }), + videoEncoder: "hevc_videotoolbox", + }), + ); + expect(graphOf(plan)).toContain("asetrate=96000,aresample=48000,atempo=0.5"); + const argv = ffmpegStep(plan).argv; + expect(argv).toEqual(expect.arrayContaining(["-tag:v", "hvc1", "-b:v", "4800k"])); + expect(argv).toContain("filter_units=remove_types=39|40"); + expect(plan.outputs.map((o) => o.kind)).toEqual(["video"]); + }); + + it("only ever uses SafeId labels and keeps text out of argv", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + overlays: [ + { id: "t1", kind: "text", atMs: 0, durationMs: 1000, text: "'; rm -rf / [0:v] %{pts}" }, + ], + }), + }), + ); + const graph = graphOf(plan); + for (const label of graph.matchAll(/\[([^\]\\]*)\]/g)) { + expect(label[1]).toMatch(/^[a-z0-9_:-]+$/); + } + expect(graph).toContain("text=\\\\\\'\\; rm -rf / \\[0\\\\:v\\] \\\\\\\\%{pts}"); + expect(ffmpegStep(plan).argv.every((arg) => !arg.includes("\0"))).toBe(true); + }); + + it("refuses unsafe or impossible input instead of guessing", () => { + expect(() => + compileRenderPlan( + input({ + edl: { + ...testEdl(), + overlays: [{ id: "BAD;ID", kind: "text", atMs: 0, durationMs: 1, text: "x" }], + } as never, + }), + ), + ).toThrow(); + expect(() => compileRenderPlan(input({ videoEncoder: "libx265" }))).toThrow( + /does not produce h264/, + ); + expect(() => compileRenderPlan(input({ sources: new Map() }))).toThrow(/no source/); + expect(() => + compileRenderPlan( + input({ + edl: testEdl({ overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 1000 }] }), + files: { ...input().files, watermarkPath: null }, + }), + ), + ).toThrow(/watermark/); + }); + + it("moves very large filtergraphs into a script file", () => { + const video = Array.from({ length: 300 }, (_, i) => ({ + id: `c${i}`, + assetId: "asset-a", + inMs: 0, + outMs: 100, + atMs: i * 100, + reframe: Array.from({ length: 8 }, (_, k) => ({ + tMs: k * 12, + cx: 0.3 + k * 0.05, + cy: 0.5, + scale: 1, + })), + })); + const plan = compileRenderPlan(input({ edl: testEdl({ tracks: { video } }) })); + const script = plan.steps.find((s) => s.id === "filtergraph"); + expect(script?.kind).toBe("write_file"); + expect(ffmpegStep(plan).argv).toContain("-filter_complex_script"); + }); +}); + +describe("escaping", () => { + it("escapes every filtergraph and drawtext special character", () => { + expect(escapeFilterText("plain text")).toBe("plain text"); + expect(escapeFilterText("a:b")).toBe("a\\\\:b"); + expect(escapeFilterText("a,b;c")).toBe("a\\,b\\;c"); + expect(escapeFilterText("[0:v]")).toBe("\\[0\\\\:v\\]"); + expect(escapeFilterText("x=y")).toBe("x\\\\=y"); + expect(escapeFilterText("it's")).toBe("it\\\\\\'s"); + expect(escapeFilterText("100%")).toBe("100\\\\\\\\%"); + expect(escapeFilterText("a\\b")).toBe("a\\\\\\\\\\\\\\\\b"); + expect(escapeFilterText(" line one\r\nline two\u0007 ")).toBe("line one\nline two"); + }); + + /** av_get_token: one parser level; returns the token and what it left unconsumed. */ + function getToken(value: string, terms: string): { token: string; rest: string } { + let token = ""; + let i = 0; + while (i < value.length && !terms.includes(value[i]!)) { + const char = value[i++]!; + if (char === "\\" && i < value.length) token += value[i++]; + else if (char === "'") { + while (i < value.length && value[i] !== "'") token += value[i++]; + i++; + } else token += char; + } + return { token, rest: value.slice(i) }; + } + + function drawtextExpand(value: string): string { + let out = ""; + for (let i = 0; i < value.length; i++) { + if (value[i] === "\\" && i + 1 < value.length) out += value[++i]; + else if (value[i] === "%") throw new Error("expansion sequence reached drawtext"); + else out += value[i]; + } + return out; + } + + it("neutralizes injection attempts", () => { + const attempts = [ + "'; rm -rf /", + "[0:v]", + "%{pts}", + "%{eif:1:d}", + "x:fontfile=/etc/passwd", + "a',drawtext=text='b", + "a\\", + "50% off; [in]null[out]", + ]; + for (const attempt of attempts) { + const graphLevel = getToken(escapeFilterText(attempt), "[],;"); + expect(graphLevel.rest).toBe(""); + const optionLevel = getToken(graphLevel.token, ":"); + expect(optionLevel.rest).toBe(""); + expect(drawtextExpand(optionLevel.token)).toBe(attempt); + } + expect(escapeFilterText("%{pts}")).toBe("\\\\\\\\%{pts}"); + expect(escapeFilterValue("C:\\media\\a,b.cube")).toBe("C\\\\:\\\\\\\\media\\\\\\\\a\\,b.cube"); + expect(() => escapeFilterValue("a\0b")).toThrow(); + }); + + it("chains atempo within its clean range", () => { + expect(atempoChain(1)).toEqual([]); + expect(atempoChain(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]); + expect(atempoChain(3)).toEqual(["atempo=2", "atempo=1.5"]); + expect(atempoChain(4)).toEqual(["atempo=2", "atempo=2"]); + }); +}); + +describe("captions", () => { + it("builds cues by line length, line count, pauses and sentence ends", () => { + const cues = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 1 }); + expect(cues.map((c) => c.text)).toEqual(["Wait for", "this!", "Here", "it is --> now."]); + const two = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 2 }); + expect(two[0]).toMatchObject({ startMs: 200, endMs: 1800, text: "Wait for\nthis!" }); + expect(two[0]!.words).toHaveLength(3); + expect( + buildCaptionCues([{ startMs: 0, endMs: 0, text: " " }], { maxCharsPerLine: 10, maxLines: 2 }), + ).toEqual([]); + const short = buildCaptionCues([{ startMs: 1000, endMs: 1100, text: "Hi" }], { + maxCharsPerLine: 10, + maxLines: 2, + }); + expect(short[0]!.endMs).toBe(1500); + }); + + it("formats SRT and VTT", () => { + const cues = buildCaptionCues(words, { maxCharsPerLine: 32, maxLines: 2 }); + expect(toSrt(cues)).toBe( + "1\n00:00:00,200 --> 00:00:01,800\nWait for this!\n\n2\n00:00:05,100 --> 00:00:06,400\nHere it is -> now.\n", + ); + expect(toVtt(cues)).toBe( + "WEBVTT\n\n00:00:00.200 --> 00:00:01.800\nWait for this!\n\n00:00:05.100 --> 00:00:06.400\nHere it <b>is</b> --> now.\n", + ); + expect(toSrt([{ startMs: 3_723_004, endMs: 3_723_500, text: "a\n\nb", words: [] }])).toBe( + "1\n01:02:03,004 --> 01:02:03,500\na\nb\n", + ); + }); + + it("maps source words to output time across speed and intro", () => { + const edl = testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1000 }, + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 1000, + outMs: 2000, + atMs: 0, + reframe: "none", + speed: 2, + }, + { + id: "c2", + assetId: "asset-a", + inMs: 5000, + outMs: 6000, + atMs: 500, + reframe: "none", + muted: true, + }, + ], + }, + }); + expect(outputCaptionWords(edl, new Map([["asset-a", words]]))).toEqual([ + { startMs: 1125, endMs: 1400, text: "this!" }, + ]); + }); + + it("writes ASS without override injection and with karaoke timing", () => { + const cues = buildCaptionCues( + [ + { startMs: 0, endMs: 400, text: "{\\pos(0,0)}hi" }, + { startMs: 600, endMs: 900, text: "you\\N" }, + ], + { maxCharsPerLine: 32, maxLines: 2 }, + ); + const ass = toAss(cues, { + width: 1080, + height: 1920, + profile: testProfile(), + preset: { + ...testKit().captionPresets[0]!, + template: "karaoke", + highlightColor: "#FFAA00", + uppercase: true, + }, + fallbackTextColor: "#FFFFFF", + position: "lower", + maxCharsPerLine: 32, + wordHighlight: true, + }); + const dialogue = ass.split("\n").find((line) => line.startsWith("Dialogue:"))!; + expect(dialogue).toBe( + "Dialogue: 0,0:00:00.00,0:00:00.90,Default,,0,0,0,,{\\k40}\uFF5B\\\u2060POS(0,0)\uFF5DHI {\\k20}{\\k30}YOU\\\u2060N", + ); + expect(ass).toContain("Style: Default,Example Sans,96,&H0000AAFF,&H00FFFFFF"); + expect(ass).toContain(",2,92,168,422,1"); + }); +}); + +describe("reframing", () => { + const geometry = { + sourceWidth: 1920, + sourceHeight: 1080, + outWidth: 1080, + outHeight: 1920, + durationMs: 4000, + }; + + it("follows the subject smoothly and keeps the crop inside the frame", () => { + const path = autoReframe( + [ + { tMs: 0, cx: 0, cy: 0.5, strength: 1 }, + { tMs: 2000, cx: 0.5, cy: 0.5, strength: 1 }, + { tMs: 4000, cx: 1, cy: 0.5, strength: 1 }, + ], + geometry, + ); + const half = 608 / 1920 / 2; + for (const k of path) { + expect(k.cx).toBeGreaterThanOrEqual(half - 1e-4); + expect(k.cx).toBeLessThanOrEqual(1 - half + 1e-4); + expect(k.cy).toBe(0.5); + expect(k.scale).toBe(1); + } + expect(path[0]!.tMs).toBe(0); + expect(path.at(-1)!.tMs).toBe(4000); + for (let i = 1; i < path.length; i++) + expect(path[i]!.cx).toBeGreaterThanOrEqual(path[i - 1]!.cx); + expect(path.length).toBeLessThanOrEqual(64); + expect(autoReframe([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]); + }); + + it("keeps faces out of the frame or reports that it cannot", () => { + const left = reframeExcluding( + [{ tMs: 100, box: { x: 0.05, y: 0.3, w: 0.15, h: 0.3 } }], + geometry, + ); + expect(left).toHaveLength(1); + const k = left![0]!; + expect(k.tMs).toBe(100); + expect(k.scale).toBe(1); + expect(k.cx - 608 / 1920 / 2).toBeGreaterThanOrEqual(0.22 - 1e-6); + // Faces all over the frame: no crop can exclude them. + const everywhere = [0, 0.25, 0.5, 0.75].map((x) => ({ + tMs: 0, + box: { x, y: 0, w: 0.25, h: 1 }, + })); + expect(reframeExcluding(everywhere, geometry)).toBeNull(); + // A face across the middle forces a tighter crop. + const wide = reframeExcluding([{ tMs: 0, box: { x: 0.2, y: 0.4, w: 0.6, h: 0.2 } }], geometry); + expect(wide![0]!.scale).toBeGreaterThan(1); + expect(reframeExcluding([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]); + }); +}); diff --git a/packages/media/src/edl.test.ts b/packages/media/src/edl.test.ts new file mode 100644 index 0000000..f72e9f9 --- /dev/null +++ b/packages/media/src/edl.test.ts @@ -0,0 +1,337 @@ +import type { EdlError } from "@rakazo/contracts"; +import { EdlErrorSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { EdlContext } from "./edl.js"; +import { edlDurationMs, locateOutputTime, locateOutputTimeAll, validateEdl } from "./edl.js"; +import { testContext, testEdl, testProfile } from "./test-fixtures.js"; + +/** Raw plan (pre-defaults) so cases can break it in any way. */ +function raw(patch: (plan: Record) => void): unknown { + const plan = JSON.parse(JSON.stringify(testEdl())) as Record; + patch(plan); + return plan; +} + +function codes(input: unknown, context: EdlContext = testContext()): EdlError["code"][] { + const result = validateEdl(input, context); + return result.ok ? [] : result.errors.map((e) => e.code); +} + +describe("validateEdl", () => { + it("accepts a valid plan and returns the parsed EDL", () => { + const result = validateEdl(testEdl(), testContext()); + expect(result.ok).toBe(true); + if (result.ok) expect(result.edl.tracks.video).toHaveLength(2); + }); + + const cases: Array<{ + code: EdlError["code"]; + input: unknown; + context?: Partial; + path?: string; + }> = [ + { code: "schema", input: raw((p) => (p.edlVersion = 2)), path: "edlVersion" }, + { + code: "schema", + input: raw((p) => (p.tracks.video[1].id = "c1")), + path: "tracks.video[1].id", + }, + { + code: "schema", + input: raw((p) => (p.tracks.video[0].id = "Bad Id;[x]")), + path: "tracks.video[0].id", + }, + { + code: "unknown_asset", + input: raw((p) => (p.tracks.video[0].assetId = "missing")), + path: "tracks.video[0].assetId", + }, + { + code: "unknown_clip", + input: raw((p) => { + p.transitions = [{ fromClipId: "c1", toClipId: "c9", kind: "cut", durationMs: 0 }]; + }), + path: "transitions[0].toClipId", + }, + { + code: "unknown_clip", + input: raw((p) => { + p.redactions.regions = [ + { + id: "r1", + clipId: "zz", + keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 0.1, h: 0.1 } }], + treatment: "blur", + }, + ]; + }), + }, + { + code: "unknown_artifact", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "sfx", artifactId: "nope", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "unknown_artifact", + input: raw((p) => (p.overlays = [{ id: "w", kind: "watermark", atMs: 0, durationMs: 1000 }])), + context: { brandKit: null }, + }, + { + code: "unknown_mark", + input: raw((p) => (p.redactions.marks = [{ markId: "mark-x", treatment: "blur" }])), + }, + { + code: "unknown_person", + input: raw((p) => { + p.redactions.faces = [{ id: "f", target: { personId: "person-x" }, mode: "blur" }]; + }), + }, + { + code: "unknown_track", + input: raw((p) => { + p.redactions.faces = [{ id: "f", target: { trackId: "track-x" }, mode: "pixelate" }]; + }), + }, + { + code: "unknown_template", + input: raw((p) => (p.intro = { templateId: "intro-none", durationMs: 1000 })), + path: "intro.templateId", + }, + { + code: "unknown_template", + input: raw((p) => (p.captions = { mode: "burned", presetId: "missing" })), + path: "captions.presetId", + }, + { + code: "range_out_of_asset", + input: raw((p) => (p.tracks.video[1].outMs = 12_000)), + path: "tracks.video[1].outMs", + }, + { + code: "range_out_of_asset", + input: raw((p) => (p.tracks.video[0].reframe = [{ tMs: 9000, cx: 0.5, cy: 0.5, scale: 1 }])), + }, + { + code: "range_empty", + input: raw((p) => (p.tracks.video[0].outMs = 0)), + }, + { + code: "range_empty", + input: raw( + (p) => (p.overlays = [{ id: "t", kind: "text", atMs: 0, durationMs: 0, text: "x" }]), + ), + }, + { + code: "overlap", + input: raw((p) => (p.tracks.video[1].atMs = 1500)), + path: "tracks.video[1].atMs", + }, + { + code: "overlap", + // A crossfade needs exactly its duration of overlap. + input: raw((p) => { + p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }]; + }), + }, + { + code: "overlap", + input: raw((p) => { + p.tracks.audio = [ + { id: "m1", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 0 }, + { id: "m2", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 2000 }, + ]; + }), + }, + { + code: "duration_below_min", + input: raw((p) => (p.output.durationHardMinMs = 5000)), + }, + { + code: "duration_below_min", + input: testEdl(), + context: { profile: testProfile({ duration: { hardMinMs: 6000, hardMaxMs: 60_000 } }) }, + }, + { + code: "duration_above_max", + input: raw((p) => (p.output.durationHardMaxMs = 3000)), + }, + { + code: "duration_above_max", + input: testEdl(), + context: { profile: testProfile({ duration: { hardMinMs: 0, hardMaxMs: 3000 } }) }, + }, + { + code: "overlay_outside_safe_zone", + input: raw((p) => { + p.overlays = [ + { + id: "t", + kind: "text", + atMs: 0, + durationMs: 1000, + text: "Hi", + box: { x: 0.1, y: 0.85, w: 0.5, h: 0.1 }, + }, + ]; + }), + path: "overlays[0].box", + }, + { + code: "paid_only_segment", + input: testEdl(), + context: { paidOnly: [{ assetId: "asset-a", startMs: 6500, endMs: 9000 }] }, + path: "tracks.video[1]", + }, + { + code: "paid_only_segment", + input: testEdl({ output: { ...testEdl().output, profileId: "example-paid-video" } }), + context: { + profile: testProfile({ id: "example-paid-video", kind: "paid_video" }), + paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 100 }], + goal: "teaser", + }, + }, + { + code: "profile_mismatch", + input: raw((p) => (p.output.profileId = "other-profile")), + }, + { + code: "profile_mismatch", + input: raw((p) => { + p.output.width = 1920; + p.output.height = 1080; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "vo-1", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-1", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + context: { profile: testProfile({ audio: { music: "platform_library_only" } }) }, + }, + ]; + + it.each(cases)("rejects $code ($path)", ({ code, input, context, path }) => { + const result = validateEdl(input, testContext(context)); + expect(result.ok).toBe(false); + if (result.ok) return; + for (const error of result.errors) expect(EdlErrorSchema.safeParse(error).success).toBe(true); + const match = result.errors.find((e) => e.code === code); + expect(match, JSON.stringify(result.errors)).toBeDefined(); + if (path) expect(match!.path).toBe(path); + }); + + it("covers every error code", () => { + const covered = new Set(cases.map((c) => c.code)); + expect([...covered].sort()).toEqual([...EdlErrorSchema.shape.code.options].sort()); + }); + + it("allows paid-only material in a full release for a paid profile and any music where allowed", () => { + const paid = testProfile({ + id: "example-paid-video", + kind: "paid_video", + audio: { music: "any" }, + }); + const plan = raw((p) => { + p.output.profileId = "example-paid-video"; + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }); + expect( + codes( + plan, + testContext({ profile: paid, paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 9000 }] }), + ), + ).toEqual([]); + }); + + it("accepts photos without a duration and a crossfade with the right overlap", () => { + const plan = raw((p) => { + p.tracks.video[0].assetId = "photo-1"; + p.tracks.video[1].atMs = 1500; + p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }]; + }); + expect(codes(plan)).toEqual([]); + }); + + it("reports every failure at once", () => { + const plan = raw((p) => { + p.tracks.video[0].assetId = "missing"; + p.tracks.video[1].outMs = 4000; + p.redactions.marks = [{ markId: "mark-x", treatment: "cover" }]; + }); + expect(codes(plan)).toEqual(["unknown_asset", "range_empty", "unknown_mark"]); + }); +}); + +describe("timeline", () => { + const edl = testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1000 }, + outro: { templateId: "outro-simple", durationMs: 500 }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" }, + { + id: "c2", + assetId: "asset-b", + inMs: 1000, + outMs: 5000, + atMs: 1500, + speed: 2, + reframe: "none", + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "dip", durationMs: 500 }], + }); + + it("measures intro, speed, transition overlap and outro", () => { + expect(edlDurationMs(edl)).toBe(1000 + 1500 + 2000 + 500); + expect(edlDurationMs(testEdl())).toBe(4000); + }); + + it("maps output time back to clip and source time", () => { + expect(locateOutputTime(edl, 500)).toBeNull(); + expect(locateOutputTime(edl, 1000)).toEqual({ clipId: "c1", assetId: "asset-a", sourceMs: 0 }); + expect(locateOutputTime(edl, 2200)).toEqual({ + clipId: "c1", + assetId: "asset-a", + sourceMs: 1200, + }); + // Inside the transition both clips are visible; the incoming one wins. + expect(locateOutputTimeAll(edl, 2700).map((h) => h.clipId)).toEqual(["c1", "c2"]); + expect(locateOutputTime(edl, 2700)).toEqual({ + clipId: "c2", + assetId: "asset-b", + sourceMs: 1400, + }); + expect(locateOutputTime(edl, 4400)).toEqual({ + clipId: "c2", + assetId: "asset-b", + sourceMs: 4800, + }); + expect(locateOutputTime(edl, 4600)).toBeNull(); + }); +}); diff --git a/packages/media/src/gate.test.ts b/packages/media/src/gate.test.ts new file mode 100644 index 0000000..acb75e3 --- /dev/null +++ b/packages/media/src/gate.test.ts @@ -0,0 +1,531 @@ +import type { FfprobeReport, GateReport } from "@rakazo/contracts"; +import { GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { edlDurationMs } from "./edl.js"; +import type { GateInput } from "./gate.js"; +import { + checkCopy, + checkTechnical, + evaluateGate, + findMetadataLeaks, + METADATA_ALLOWED_TAGS, + planAutoFix, +} from "./gate.js"; +import { testEdl, testPaidProfile, testProfile } from "./test-fixtures.js"; + +const cleanTags = { + "format.major_brand": "isom", + "format.minor_version": "512", + "format.compatible_brands": "isomiso2avc1mp41", + "format.title": "Example", + "stream.0.language": "und", + "stream.0.handler_name": "VideoHandler", + "stream.0.vendor_id": "[0][0][0][0]", + "stream.0.encoder": "Lavc libx264", + "stream.1.handler_name": "SoundHandler", +}; + +function probe(overrides: Partial = {}): FfprobeReport { + return { + durationMs: 4000, + width: 1080, + height: 1920, + fps: 30, + videoCodec: "h264", + audioCodec: "aac", + sizeBytes: 2_000_000, + integratedLufs: -14.3, + truePeakDb: -1.2, + tags: cleanTags, + ...overrides, + }; +} + +function gateInput(overrides: Partial = {}): GateInput { + return { + profile: testProfile(), + edl: testEdl(), + ffprobe: probe(), + content: [ + { tMs: 0, scores: { explicit_nudity: 0.01 } }, + { tMs: 500, scores: { explicit_nudity: 0.02, violence: 0.1 } }, + ], + faces: [], + trackingGaps: [], + marks: [], + people: [], + copy: { title: "A day out", caption: "Short caption #one", tags: ["two"] }, + aiAlterations: [], + bundleHasDisclosure: false, + watermarkBox: { x: 0.7, y: 0.12, w: 0.15, h: 0.05 }, + sampledFrames: 8, + autoFixRound: 0, + now: new Date("2026-10-01T12:00:00Z"), + ...overrides, + }; +} + +const blockCodes = (report: GateReport) => + report.issues.filter((i) => i.severity === "block").map((i) => i.code); + +describe("metadata", () => { + it("allows only harmless container tags", () => { + expect(METADATA_ALLOWED_TAGS).toContain("title"); + expect(findMetadataLeaks(cleanTags)).toEqual([]); + expect( + findMetadataLeaks({ + ...cleanTags, + "format.creation_time": "2026-01-01T00:00:00Z", + "format.location": "+00.0000+000.0000/", + "format.com.apple.quicktime.make": "Example", + "format.com.apple.quicktime.location.ISO6709": "+00+000/", + "format.make": "Example", + "format.model": "Example", + "format.software": "Example 1.0", + "stream.0.creation_time": "2026-01-01T00:00:00Z", + GPSLatitude: "0", + "format.encoder": "Lavf60.16.100", + "stream:0:encoder": "x264 core 164", + }), + ).toEqual([ + "GPSLatitude", + "format.com.apple.quicktime.location.ISO6709", + "format.com.apple.quicktime.make", + "format.creation_time", + "format.encoder", + "format.location", + "format.make", + "format.model", + "format.software", + "stream.0.creation_time", + "stream:0:encoder", + ]); + expect(findMetadataLeaks({ title: "x", major_brand: "isom" })).toEqual([]); + }); +}); + +describe("evaluateGate", () => { + it("passes a clean render and produces a schema-valid report", () => { + const report = evaluateGate(gateInput()); + expect(report.issues).toEqual([]); + expect(report.passed).toBe(true); + expect(GateReportSchema.parse(report)).toEqual(report); + expect(report.checkedAt).toBe("2026-10-01T12:00:00.000Z"); + }); + + it("blocks technical problems", () => { + const edl = testEdl(); + expect(checkTechnical(testProfile(), edl, probe())).toEqual([]); + const cases: Array<[Partial, string]> = [ + [{ width: 1920, height: 1080 }, "resolution_mismatch"], + [{ width: 1920, height: 1080 }, "aspect_not_allowed"], + [{ fps: 25 }, "fps_mismatch"], + [{ durationMs: 70_000 }, "duration_too_long"], + [{ durationMs: 500 }, "duration_too_short"], + [{ sizeBytes: 600_000_000 }, "file_too_large"], + [{ videoCodec: "vp9" }, "codec_mismatch"], + [{ integratedLufs: -20 }, "loudness_off_target"], + [{ integratedLufs: null }, "loudness_unmeasured"], + [{ truePeakDb: 0 }, "true_peak_exceeded"], + [{ tags: { ...cleanTags, "format.creation_time": "2020" } }, "metadata_leak"], + ]; + for (const [patch, code] of cases) { + const report = evaluateGate(gateInput({ ffprobe: probe(patch) })); + expect(report.passed, code).toBe(false); + expect(blockCodes(report), code).toContain(code); + } + const drift = checkTechnical(testProfile(), edl, probe({ durationMs: 4500 })); + expect(drift).toEqual([expect.objectContaining({ code: "duration_drift", severity: "warn" })]); + expect( + checkTechnical(testProfile(), edl, probe({ audioCodec: null, integratedLufs: null })), + ).toEqual([]); + }); + + it("blocks classifier hits with timestamps, category and score", () => { + const report = evaluateGate( + gateInput({ + content: [ + { tMs: 0, scores: { explicit_nudity: 0.05 } }, + { tMs: 1500, scores: { explicit_nudity: 0.4 } }, + { tMs: 2000, scores: { explicit_nudity: 0.6 } }, + { tMs: 3500, scores: { violence: 0.9, drugs: 1 } }, + ], + }), + ); + expect(report.passed).toBe(false); + expect(report.issues).toEqual([ + expect.objectContaining({ + check: "content", + code: "content_threshold", + category: "explicit_nudity", + atMs: 1500, + endMs: 2000, + score: 0.6, + }), + expect.objectContaining({ category: "violence", atMs: 3500, severity: "block" }), + ]); + }); + + it("only blocks above the ratio when the profile says so", () => { + const profile = testProfile({ + content: { thresholds: { violence: 0.5 }, blockOn: "ratio", blockRatio: 0.25 }, + }); + const samples = Array.from({ length: 10 }, (_, i) => ({ + tMs: i * 500, + scores: { violence: i < 2 ? 0.9 : 0 }, + })); + const below = evaluateGate(gateInput({ profile, content: samples })); + expect(below.passed).toBe(true); + expect(below.issues[0]).toMatchObject({ severity: "warn", category: "violence" }); + const above = samples.map((s, i) => ({ ...s, scores: { violence: i < 4 ? 0.9 : 0 } })); + expect(evaluateGate(gateInput({ profile, content: above })).passed).toBe(false); + }); + + it("blocks anonymization gaps, uncovered and unknown faces, and visible marks", () => { + const people = [ + { id: "person-1", anonymizeByDefault: "blur", consentRecordedAt: null, ageVerifiedAt: null }, + { id: "person-2", anonymizeByDefault: "none", consentRecordedAt: null, ageVerifiedAt: null }, + ]; + const box = { x: 0.4, y: 0.2, w: 0.1, h: 0.1 }; + const report = evaluateGate( + gateInput({ + people, + faces: [ + { tMs: 100, box, personId: "person-1", coverage: 1 }, + { tMs: 200, box, personId: "person-1", coverage: 0.9 }, + { tMs: 300, box, personId: "person-2", coverage: 0 }, + { tMs: 400, box, personId: null, coverage: 0 }, + ], + trackingGaps: [ + { startMs: 1000, endMs: 1400, personId: "person-1" }, + { startMs: 2000, endMs: 2600, personId: "person-1" }, + ], + marks: [ + { markId: "mark-1", tMs: 100, residualVisibility: 0.01 }, + { markId: "mark-1", tMs: 300, residualVisibility: 0.3 }, + ], + }), + ); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([ + expect.objectContaining({ code: "face_not_covered", personId: "person-1", atMs: 200 }), + expect.objectContaining({ code: "unknown_face", atMs: 400 }), + expect.objectContaining({ + code: "tracking_gap", + atMs: 2000, + endMs: 2600, + personId: "person-1", + }), + expect.objectContaining({ code: "mark_visible", markId: "mark-1", atMs: 300 }), + ]); + // Without protected people an unmatched face is not an anonymization issue. + const open = evaluateGate(gateInput({ faces: [{ tMs: 0, box, personId: null, coverage: 0 }] })); + expect(open.passed).toBe(true); + }); + + it("requires consent and age verification only where the profile does", () => { + const people = [ + { + id: "p-ok", + anonymizeByDefault: "none", + consentRecordedAt: "2026-01-01T00:00:00Z", + ageVerifiedAt: "2026-01-01T00:00:00Z", + }, + { + id: "p-no-consent", + anonymizeByDefault: "none", + consentRecordedAt: null, + ageVerifiedAt: "2026-01-01T00:00:00Z", + }, + { + id: "p-no-age", + anonymizeByDefault: "none", + consentRecordedAt: "2026-01-01T00:00:00Z", + ageVerifiedAt: null, + }, + ]; + const faces = [{ tMs: 1000, box: { x: 0, y: 0, w: 0.1, h: 0.1 }, personId: null, coverage: 0 }]; + expect(evaluateGate(gateInput({ people, faces })).passed).toBe(true); + const profile = testPaidProfile(); + const edl = testEdl({ + output: { ...testEdl().output, profileId: profile.id, width: 1920, height: 1080 }, + }); + const report = evaluateGate( + gateInput({ profile, edl, people, faces, ffprobe: probe({ width: 1920, height: 1080 }) }), + ); + expect(report.issues.filter((i) => i.check === "consent")).toEqual([ + expect.objectContaining({ code: "age_unverified", personId: "p-no-age" }), + expect.objectContaining({ code: "consent_missing", personId: "p-no-consent" }), + expect.objectContaining({ code: "unknown_performer", atMs: 1000 }), + ]); + expect(report.passed).toBe(false); + }); + + it("requires the AI label when the profile does and warns about the watermark", () => { + const report = evaluateGate( + gateInput({ + aiAlterations: ["synthetic_identity"], + watermarkBox: { x: 0.9, y: 0.01, w: 0.1, h: 0.05 }, + }), + ); + expect(report.issues).toEqual([ + expect.objectContaining({ check: "disclosure", code: "ai_label_missing", severity: "block" }), + expect.objectContaining({ + check: "brand", + code: "watermark_outside_safe_zone", + severity: "warn", + }), + ]); + expect( + evaluateGate(gateInput({ aiAlterations: ["inpaint"], bundleHasDisclosure: true })).passed, + ).toBe(true); + const edl = testEdl({ metadata: { aiAltered: true, aiAlterations: ["voice_disguise"] } }); + expect(blockCodes(evaluateGate(gateInput({ edl })))).toEqual(["ai_label_missing"]); + const relaxed = testProfile({ disclosure: { aiAlteredLabelRequired: false } }); + expect(evaluateGate(gateInput({ profile: relaxed, edl })).passed).toBe(true); + }); +}); + +describe("checkCopy", () => { + const profile = testProfile({ + text: { + titleMaxChars: 20, + captionMaxChars: 60, + hashtagLimit: 2, + linkPolicy: "bio_only", + bannedPatterns: [ + { pattern: "free\\s+money", reason: "misleading claim" }, + { pattern: "([a-z", reason: "broken pattern" }, + ], + }, + }); + + it("checks banned patterns case-insensitively and reports invalid ones without crashing", () => { + const issues = checkCopy(profile, { + title: "FREE Money", + caption: null, + tags: ["free money"], + }); + expect(issues.map((i) => [i.code, i.message])).toEqual([ + ["banned_pattern", "title: misleading claim"], + ["banned_pattern", "tags[0]: misleading claim"], + ["invalid_banned_pattern", "profile pattern cannot be checked: broken pattern"], + ]); + }); + + it("checks lengths, hashtags and links", () => { + const codes = (copy: { title: string | null; caption: string | null; tags: string[] }) => + checkCopy(profile, copy) + .map((i) => i.code) + .filter((c) => c !== "invalid_banned_pattern"); + expect(codes({ title: "x".repeat(21), caption: "y".repeat(61), tags: [] })).toEqual([ + "title_too_long", + "caption_too_long", + ]); + expect(codes({ title: null, caption: "#a #b", tags: ["#c"] })).toEqual(["too_many_hashtags"]); + expect(codes({ title: null, caption: "#a #A", tags: ["a"] })).toEqual([]); + expect(codes({ title: null, caption: "see https://example.com/x", tags: [] })).toEqual([ + "link_bio_only", + ]); + expect(codes({ title: null, caption: "visit example.shop today", tags: [] })).toEqual([ + "link_bio_only", + ]); + expect(codes({ title: "Link in bio", caption: "Ready? Go.", tags: [] })).toEqual([]); + const forbidden = testProfile({ + text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "forbidden" }, + }); + expect( + checkCopy(forbidden, { title: "www.example.com", caption: null, tags: [] })[0]!.code, + ).toBe("link_forbidden"); + const allowed = testProfile({ + text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "allowed" }, + }); + expect(checkCopy(allowed, { title: null, caption: "https://example.com", tags: [] })).toEqual( + [], + ); + }); +}); + +describe("planAutoFix", () => { + const edl = testEdl({ + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 0, + outMs: 2000, + atMs: 0, + reframe: "auto", + segmentId: "s1", + }, + { + id: "c2", + assetId: "asset-a", + inMs: 5000, + outMs: 7000, + atMs: 2000, + reframe: "auto", + segmentId: "s2", + }, + { + id: "c3", + assetId: "asset-a", + inMs: 8000, + outMs: 10_000, + atMs: 4000, + reframe: "auto", + segmentId: "s3", + }, + ], + }, + transitions: [{ fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }], + overlays: [ + { id: "wm", kind: "watermark", atMs: 0, durationMs: 6000 }, + { id: "cta", kind: "cta_end_card", atMs: 4500, durationMs: 1500, text: "Link in bio" }, + ], + redactions: { + faces: [{ id: "f1", target: { personId: "person-1" }, mode: "blur", paddingPct: 20 }], + marks: [{ markId: "mark-1", treatment: "inpaint" }], + }, + metadata: { aiAltered: true, aiAlterations: ["inpaint"] }, + }); + const report = (issues: GateReport["issues"], round = 0): GateReport => ({ + profileId: "example-short-vertical", + passed: !issues.some((i) => i.severity === "block"), + checkedAt: "2026-10-01T12:00:00.000Z", + sampledFrames: 12, + issues, + autoFixRound: round, + }); + const contentHit = (atMs: number, endMs = atMs) => + ({ + check: "content", + severity: "block", + code: "content_threshold", + message: "violence", + atMs, + endMs, + category: "violence", + score: 0.9, + }) as const; + + const assertNoNewOverlays = (fixed: typeof edl) => { + const before = new Map(edl.overlays.map((o) => [o.id, o])); + for (const overlay of fixed.overlays) { + const original = before.get(overlay.id); + expect(original, `overlay ${overlay.id} was added`).toBeDefined(); + expect(overlay.kind).toBe(original!.kind); + expect(overlay.text).toBe(original!.text); + expect(overlay.durationMs).toBeLessThanOrEqual(original!.durationMs); + } + expect(fixed.overlays.length).toBeLessThanOrEqual(edl.overlays.length); + }; + + it("swaps an offending clip for a clean alternative segment", () => { + const fix = planAutoFix(edl, report([contentHit(2500, 3000)]), [ + { segmentId: "s2", assetId: "asset-a", startMs: 5000, endMs: 7000 }, + { segmentId: "s9", assetId: "asset-a", startMs: 5500, endMs: 9000 }, + { segmentId: "s8", assetId: "asset-b", startMs: 1000, endMs: 4000 }, + ])!; + expect(fix.changes).toEqual(["clip c2: swapped to segment s8 (content_threshold)"]); + expect(fix.edl.tracks.video.find((c) => c.id === "c2")).toMatchObject({ + assetId: "asset-b", + inMs: 1000, + outMs: 3000, + segmentId: "s8", + atMs: 2000, + }); + expect(edlDurationMs(fix.edl)).toBe(6000); + assertNoNewOverlays(fix.edl); + }); + + it("trims or drops the clip when no alternative fits, and repacks the timeline", () => { + const trimmed = planAutoFix(edl, report([contentHit(3800)]), [])!; + expect(trimmed.changes).toEqual(["clip c2: trimmed the end at 6300 ms (content_threshold)"]); + expect(trimmed.edl.tracks.video.map((c) => [c.id, c.atMs, c.inMs, c.outMs])).toEqual([ + ["c1", 0, 0, 2000], + ["c2", 2000, 5000, 6300], + ["c3", 3300, 8000, 10_000], + ]); + assertNoNewOverlays(trimmed.edl); + expect(trimmed.edl.overlays.find((o) => o.id === "wm")!.durationMs).toBe(5300); + + const dropped = planAutoFix(edl, report([contentHit(2300, 3700)]), [])!; + expect(dropped.changes).toEqual(["clip c2: removed (content_threshold)"]); + expect(dropped.edl.tracks.video.map((c) => c.id)).toEqual(["c1", "c3"]); + expect(dropped.edl.transitions).toEqual([]); + assertNoNewOverlays(dropped.edl); + }); + + it("cuts tracking gaps, widens face redactions, adds a blur for unknown faces, and covers marks", () => { + const fix = planAutoFix( + edl, + report([ + { + check: "anonymization", + severity: "block", + code: "face_not_covered", + message: "", + atMs: 100, + personId: "person-1", + }, + { check: "anonymization", severity: "block", code: "unknown_face", message: "", atMs: 300 }, + { + check: "anonymization", + severity: "block", + code: "mark_visible", + message: "", + atMs: 300, + markId: "mark-1", + }, + { + check: "anonymization", + severity: "block", + code: "tracking_gap", + message: "", + atMs: 4100, + endMs: 4800, + }, + ]), + [], + )!; + expect(fix.edl.redactions.faces).toEqual([ + expect.objectContaining({ id: "f1", paddingPct: 45 }), + expect.objectContaining({ + id: "fix-face-unassigned", + target: { unassignedFaces: true }, + mode: "blur", + }), + ]); + expect(fix.edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "cover" }]); + expect(fix.edl.metadata.aiAlterations).toEqual([]); + expect(fix.edl.tracks.video.find((c) => c.id === "c3")).toMatchObject({ inMs: 9300 }); + assertNoNewOverlays(fix.edl); + }); + + it("stops at the round limit, on passing reports, and when nothing safe applies", () => { + expect(planAutoFix(edl, report([contentHit(2500)], MAX_AUTO_FIX_ROUNDS), [])).toBeNull(); + expect(planAutoFix(edl, report([]), [])).toBeNull(); + const textOnly = report([ + { check: "text", severity: "block", code: "banned_pattern", message: "" }, + ]); + expect(planAutoFix(edl, textOnly, [])).toBeNull(); + const consent = report([ + { check: "consent", severity: "block", code: "consent_missing", message: "", personId: "p" }, + ]); + expect(planAutoFix(edl, consent, [])).toBeNull(); + }); + + it("runs at most three rounds in a loop", () => { + let current = edl; + let rounds = 0; + for (let round = 0; round < 10; round++) { + const fix = planAutoFix(current, report([contentHit(500)], round), []); + if (!fix) break; + current = fix.edl; + rounds += 1; + assertNoNewOverlays(current); + } + expect(rounds).toBe(MAX_AUTO_FIX_ROUNDS); + }); +}); diff --git a/packages/media/src/marketing.test.ts b/packages/media/src/marketing.test.ts new file mode 100644 index 0000000..f2d43fc --- /dev/null +++ b/packages/media/src/marketing.test.ts @@ -0,0 +1,320 @@ +import type { MetricCsvMapping, MotionTemplateInfo } from "@rakazo/contracts"; +import { MetricCsvMappingSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { + buildTrackingUrl, + contrastRatio, + introFits, + normalizeHashtags, + parseCsvRecords, + parseMetricsCsv, + parseMoneyToCents, + proposeIntroVariants, + uploadChecklist, + validateBrandKit, +} from "./marketing.js"; +import { testKit, testPaidProfile, testProfile } from "./test-fixtures.js"; + +const utm = { source: "example-short-vertical", medium: "short", campaign: "spring launch" }; + +describe("buildTrackingUrl", () => { + it("appends UTM parameters and keeps the existing query and fragment", () => { + expect(buildTrackingUrl("https://example.com/shop", utm)).toBe( + "https://example.com/shop?utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch", + ); + expect( + buildTrackingUrl("https://example.com/p?ref=a%20b&x=1&utm_source=old#top", { + ...utm, + content: "thumb-2", + term: "ä&b", + }), + ).toBe( + "https://example.com/p?ref=a%20b&x=1&utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch&utm_content=thumb-2&utm_term=%C3%A4%26b#top", + ); + expect(buildTrackingUrl("http://example.com", utm)).toMatch( + /^http:\/\/example\.com\/\?utm_source=/, + ); + }); + + it("rejects non-http schemes and invalid URLs", () => { + expect(() => buildTrackingUrl("javascript:alert(1)", utm)).toThrow(/http/); + expect(() => buildTrackingUrl("ftp://example.com", utm)).toThrow(/http/); + expect(() => buildTrackingUrl("not a url", utm)).toThrow(/valid/); + }); +}); + +describe("normalizeHashtags", () => { + it("normalizes, dedupes and limits", () => { + expect( + normalizeHashtags(["#Summer", "summer", "##new day", "über-cool!", "", "#"], 10), + ).toEqual(["#Summer", "#newday", "#übercool"]); + expect(normalizeHashtags(["a", "b", "c"], 2)).toEqual(["#a", "#b"]); + expect(normalizeHashtags(["a"], 0)).toEqual([]); + }); +}); + +describe("CSV", () => { + const mapping = (overrides: Partial = {}): MetricCsvMapping => + MetricCsvMappingSchema.parse({ + name: "Example", + columns: { + publication: "Link", + observedAt: "Date", + views: "Views", + clicks: "Clicks", + conversions: "Sales", + revenue: "Revenue", + }, + publicationKey: "tracking_url", + ...overrides, + }); + + it("parses RFC 4180 records with quotes, delimiters and newlines in fields", () => { + expect(parseCsvRecords('a,"b,c","say ""hi""\nthere"\r\n1,2,3\n', ",")).toEqual([ + ["a", "b,c", 'say "hi"\nthere'], + ["1", "2", "3"], + ]); + expect(parseCsvRecords("x;y\n", ";")).toEqual([["x", "y"]]); + expect(parseCsvRecords("", ",")).toEqual([]); + }); + + it("imports metrics with BOM, comma delimiter and major-unit revenue", () => { + const csv = + 'Link,Date,Views,Clicks,Sales,Revenue\n"https://example.com/?utm_content=a,b",2026-09-01,"1,200",30,2,"19.98"\n'; + const result = parseMetricsCsv(csv, mapping()); + expect(result.skipped).toEqual([]); + expect(result.rows).toEqual([ + { + row: 2, + publicationKey: "https://example.com/?utm_content=a,b", + observedAt: new Date("2026-09-01T00:00:00Z"), + views: 1200, + clicks: 30, + conversions: 2, + revenueCents: 1998, + currency: "EUR", + }, + ]); + }); + + it("handles semicolons, tabs, decimal commas, currencies and skipped rows", () => { + const semicolon = [ + "link;date;views;clicks;sales;revenue;currency", + "pub-1;01.09.2026;1.500;12;1;1.234,50 €;", + "pub-2;2026/09/02;10;1;0;12,5;usd", + "pub-3;31.02.2026;1;0;0;0;", + "pub-4;2026-09-03;ten;0;0;0;", + ";2026-09-03;1;0;0;0;", + "pub-5;2026-09-04;1;0;0;-5;", + "pub-6;2026-09-04;1;0;0;1,2,3;", + "pub-7;2026-09-04;1;0;0;5;EURO", + ";;;;;;", + "pub-8;2026-09-05T10:30:00Z;7;;;$3.10;", + ].join("\r\n"); + const result = parseMetricsCsv( + semicolon, + mapping({ + delimiter: ";", + columns: { + publication: "LINK", + observedAt: "date", + views: "views", + clicks: "clicks", + conversions: "sales", + revenue: "revenue", + currency: "currency", + }, + }), + ); + expect(result.rows.map((r) => [r.publicationKey, r.views, r.revenueCents, r.currency])).toEqual( + [ + ["pub-1", 1500, 123_450, "EUR"], + ["pub-2", 10, 1250, "USD"], + ["pub-8", 7, 310, "USD"], + ], + ); + expect(result.rows[2]!.observedAt).toEqual(new Date("2026-09-05T10:30:00Z")); + expect(result.skipped).toEqual([ + { row: 4, reason: "date is not recognized" }, + { row: 5, reason: "views is not a whole number" }, + { row: 6, reason: "publication is empty" }, + { row: 7, reason: "revenue is negative" }, + { row: 8, reason: "revenue is not a number" }, + { row: 9, reason: "currency is not a 3-letter code" }, + ]); + + const tabs = parseMetricsCsv( + "Link\tDate\tRevenue\nx\t2026-09-01\t250\n", + mapping({ + delimiter: "\t", + revenueUnit: "cents", + columns: { publication: "Link", observedAt: "Date", revenue: "Revenue" }, + }), + ); + expect(tabs.rows[0]).toMatchObject({ views: 0, clicks: 0, conversions: 0, revenueCents: 250 }); + }); + + it("reports missing columns and empty files", () => { + expect(parseMetricsCsv("Other,Date\n", mapping())).toEqual({ + rows: [], + skipped: [{ row: 1, reason: "missing columns: Link, Views, Clicks, Sales, Revenue" }], + }); + expect(parseMetricsCsv("", mapping()).skipped).toEqual([{ row: 1, reason: "file is empty" }]); + }); + + it("converts money in either notation to cents", () => { + expect(parseMoneyToCents("1,234.56")).toEqual({ cents: 123_456, currency: null }); + expect(parseMoneyToCents("1.234,56")).toEqual({ cents: 123_456, currency: null }); + expect(parseMoneyToCents("1,234")).toEqual({ cents: 123_400, currency: null }); + expect(parseMoneyToCents("0.995")).toEqual({ cents: 100, currency: null }); + expect(parseMoneyToCents("2.5")).toEqual({ cents: 250, currency: null }); + expect(parseMoneyToCents("CHF 7")).toEqual({ cents: 700, currency: "CHF" }); + expect(parseMoneyToCents("12.345.678,9")).toEqual({ cents: 1_234_567_890, currency: null }); + expect(parseMoneyToCents("1,5,0")).toBeNull(); + }); +}); + +describe("intro", () => { + const templates: MotionTemplateInfo[] = [ + { + id: "intro-glitch", + kind: "intro", + name: "Glitch energy", + minDurationMs: 1000, + maxDurationMs: 5000, + params: [{ name: "primaryColor", type: "color", required: true }], + }, + { + id: "intro-elegant", + kind: "intro", + name: "Elegant fade", + minDurationMs: 2000, + maxDurationMs: 3000, + params: [ + { name: "brandName", type: "string", required: true }, + { name: "sound", type: "string", required: false }, + ], + }, + { + id: "intro-kinetic", + kind: "intro", + name: "Kinetic type", + minDurationMs: 1500, + maxDurationMs: 4000, + params: [], + }, + { + id: "intro-logo", + kind: "intro", + name: "Minimal logo reveal", + minDurationMs: 1500, + maxDurationMs: 4000, + params: [{ name: "logoArtifactId", type: "string", required: true }], + }, + { + id: "outro-a", + kind: "outro", + name: "Outro", + minDurationMs: 1000, + maxDurationMs: 3000, + params: [], + }, + ]; + + it("keeps shorts hook-first and allows short intros on long-form profiles", () => { + expect(introFits(testProfile(), 1500)).toBe(false); + expect(introFits(testProfile(), 0)).toBe(true); + expect(introFits(testPaidProfile(), 3000)).toBe(true); + expect(introFits(testPaidProfile(), 5000)).toBe(false); + }); + + it("proposes three distinct variants ordered by tone", () => { + const kit = testKit({ tone: "calm, premium", logoArtifactIds: ["logo-1"] }); + const variants = proposeIntroVariants(kit, templates, 3); + expect(variants.map((v) => v.template.templateId)).toEqual([ + "intro-elegant", + "intro-logo", + "intro-glitch", + ]); + expect(new Set(variants.map((v) => v.template.templateId)).size).toBe(3); + for (const variant of variants) { + expect(variant.template.durationMs).toBeGreaterThanOrEqual(1500); + expect(variant.template.durationMs).toBeLessThanOrEqual(4000); + } + expect(variants[0]!.template.params).toEqual({ brandName: "Studio Example", sound: "music-1" }); + expect(variants[0]!.rationale).toContain('matches the tone "calm, premium"'); + expect(variants[1]!.template.params).toEqual({ logoArtifactId: "logo-1" }); + expect(proposeIntroVariants(kit, templates, 3)).toEqual(variants); + const energetic = proposeIntroVariants(testKit({ tone: "bold and energetic" }), templates, 3); + expect(energetic[0]!.template).toMatchObject({ + templateId: "intro-glitch", + params: { primaryColor: "#112233" }, + }); + expect(proposeIntroVariants(kit, [], 3)).toEqual([]); + expect( + proposeIntroVariants(kit, templates.slice(0, 1), 3).map((v) => v.template.durationMs), + ).toEqual([2500, 3000, 1500]); + }); +}); + +describe("validateBrandKit", () => { + it("flags low contrast, duplicate presets, unlicensed music and fonts", () => { + expect(validateBrandKit(testKit())).toEqual([]); + expect(contrastRatio("#FFFFFF", "#000000")).toBeCloseTo(21, 5); + const preset = testKit().captionPresets[0]!; + const warnings = validateBrandKit( + testKit({ + colors: { ...testKit().colors, text: "#777777", background: "#888888" }, + captionPresets: [preset, { ...preset, backgroundColor: "#EEEEEE" }], + musicBeds: [ + { artifactId: "m", title: "Borrowed", license: "unknown", commercialUse: false }, + ], + fonts: [{ family: "Nameless", artifactId: "f", role: "body" }], + }), + ); + expect(warnings).toEqual([ + "text on background has contrast 1.3:1, below 4.5:1", + 'caption preset id "clean" is used twice', + 'caption preset "clean" has contrast 1.2:1, below 4.5:1', + 'music bed "Borrowed" is not licensed for commercial use', + 'font "Nameless" has no license note', + ]); + }); +}); + +describe("uploadChecklist", () => { + it("lists the AI label, age restriction, link policy and captions where they apply", () => { + const paid = uploadChecklist({ + profile: testPaidProfile(), + aiAltered: true, + hasCaptions: true, + trackingUrl: "https://example.com/?utm_source=x", + }); + expect(paid).toContain("# Upload checklist: Example paid video"); + expect(paid).toContain("AI-generated or altered content label"); + expect(paid).toContain("age-restricted"); + expect(paid).toContain("Add the tracking link: "); + expect(paid).toContain("SRT or VTT"); + + const short = uploadChecklist({ + profile: testProfile({ captions: { recommended: true, required: true } }), + aiAltered: false, + hasCaptions: false, + trackingUrl: "https://example.com/", + }); + expect(short).not.toContain("AI-generated"); + expect(short).not.toContain("age-restricted"); + expect(short).toContain("in the profile bio, not in the caption"); + expect(short).toContain("Captions are required"); + expect(short).toContain("at most 3 hashtags"); + expect(short.split("\n").filter((l) => l.startsWith("- [ ] "))).toHaveLength(4); + + const relaxed = uploadChecklist({ + profile: testProfile({ disclosure: { aiAlteredLabelRequired: false } }), + aiAltered: true, + hasCaptions: true, + trackingUrl: null, + }); + expect(relaxed).not.toContain("AI-generated"); + }); +}); diff --git a/packages/media/src/teaser.test.ts b/packages/media/src/teaser.test.ts new file mode 100644 index 0000000..e2f218c --- /dev/null +++ b/packages/media/src/teaser.test.ts @@ -0,0 +1,334 @@ +import { + MediaProjectGoalSchema, + TEASER_FEATURES, + TEASER_LEARNING_MIN_SAMPLES, +} from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { validateEdl } from "./edl.js"; +import type { ScoredSegmentInput, SegmentFeatures } from "./teaser.js"; +import { + autoPlanEdl, + defaultTeaserWeights, + fitTeaserWeights, + rankThumbnailFrames, + scoreSegment, + selectTeaserSegments, + speechHookScore, +} from "./teaser.js"; +import { testContext, testKit, testPaidProfile, testProfile } from "./test-fixtures.js"; + +function segment( + id: string, + assetId: string, + startMs: number, + endMs: number, + features: SegmentFeatures, + extra: Partial = {}, +): ScoredSegmentInput { + return { + id, + assetId, + startMs, + endMs, + capturedAt: null, + features, + paidOnly: false, + transcript: null, + ...extra, + }; +} + +/** One 30 s asset building up to a climax at 20–24 s, plus a second asset. */ +const segments: ScoredSegmentInput[] = [ + segment("s1", "asset-a", 0, 4000, { motion: 0.2, sharpness: 0.8 }), + segment( + "s2", + "asset-a", + 4000, + 8000, + { motion: 0.3, audioPeak: 0.3 }, + { transcript: "Wait, you won't believe what happens next?" }, + ), + segment("s3", "asset-a", 8000, 12_000, { motion: 0.5, faces: 0.6 }), + segment("s4", "asset-a", 12_000, 16_000, { motion: 0.6, audioPeak: 0.5 }), + segment("s5", "asset-a", 16_000, 20_000, { motion: 0.7, novelty: 0.6 }, { paidOnly: true }), + segment("s6", "asset-a", 20_000, 24_000, { + motion: 1, + audioPeak: 1, + faces: 1, + novelty: 1, + aesthetics: 1, + }), + segment("s7", "asset-a", 24_000, 28_000, { motion: 0.4 }), + segment("s8", "asset-b", 0, 3000, { motion: 0.5, aesthetics: 0.7 }), +]; + +describe("weights and scores", () => { + it("defines every feature for every goal", () => { + for (const goal of MediaProjectGoalSchema.options) { + const weights = defaultTeaserWeights(goal); + expect(Object.keys(weights).sort()).toEqual([...TEASER_FEATURES].sort()); + expect(Object.values(weights).reduce((a, b) => a + b, 0)).toBeCloseTo(1, 6); + } + expect(defaultTeaserWeights("teaser").speechHook).toBeGreaterThan( + defaultTeaserWeights("highlight").speechHook, + ); + }); + + it("scores as a weighted mean of clamped features", () => { + const weights = defaultTeaserWeights("teaser"); + expect(scoreSegment({}, weights)).toBe(0); + const all = Object.fromEntries(TEASER_FEATURES.map((f) => [f, 1])); + expect(scoreSegment(all, weights)).toBeCloseTo(1, 6); + expect(scoreSegment({ motion: 5 }, weights)).toBeCloseTo(weights.motion, 6); + expect(scoreSegment({ motion: Number.NaN }, weights)).toBe(0); + expect(scoreSegment({ motion: 1 }, { ...weights, motion: 0 })).toBe(0); + }); + + it("detects German and English speech hooks", () => { + expect(speechHookScore("")).toBe(0); + expect(speechHookScore("the weather was fine")).toBe(0); + expect(speechHookScore("Wait, you won't believe this!", "en")).toBeGreaterThan(0.6); + expect(speechHookScore("Warum macht das niemand?", "de")).toBeGreaterThan(0.5); + expect(speechHookScore("Schau mal, 3 Tricks!", "de-DE")).toBeGreaterThan(0.6); + expect(speechHookScore("Here are 5 tips")).toBeGreaterThan(speechHookScore("here are tips")); + expect(speechHookScore("Wie geht das? Pass auf! 10 Sekunden, du!")).toBe(1); + }); +}); + +describe("selectTeaserSegments", () => { + const profile = testProfile(); + const select = (overrides: Partial[0]> = {}) => + selectTeaserSegments({ + segments, + profile, + goal: "teaser", + targetMs: 15_000, + weights: defaultTeaserWeights("teaser"), + endCardMs: 2000, + ...overrides, + }); + + it("opens with the hook inside the hook window", () => { + const selection = select(); + expect(selection.hookSegmentId).toBe("s2"); + expect(selection.clips[0]).toMatchObject({ segmentId: "s2", inMs: 4000 }); + expect(selection.notes[0]).toContain("hook window"); + }); + + it("never uses paid-only segments and ends before the payoff", () => { + const selection = select(); + const ids = selection.clips.map((c) => c.segmentId); + expect(ids).not.toContain("s5"); + expect(ids).not.toContain("s6"); + expect(ids).not.toContain("s7"); + const last = selection.clips.at(-1)!; + expect(last.assetId).toBe("asset-a"); + expect(last.outMs).toBeLessThanOrEqual(20_000); + expect(selection.notes.some((n) => n.startsWith("open loop"))).toBe(true); + }); + + it("builds tension and fits the duration bounds minus the end card", () => { + const selection = select(); + const total = selection.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0); + expect(total).toBeLessThanOrEqual(15_000 - 2000); + expect(total).toBeGreaterThanOrEqual(profile.duration.hardMinMs - 2000); + for (const clip of selection.clips) expect(clip.outMs).toBeGreaterThan(clip.inMs); + const tight = select({ targetMs: 5000 }); + expect(tight.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0)).toBeLessThanOrEqual(3000); + }); + + it("returns nothing when only paid-only material exists", () => { + const paid = segments.map((s) => ({ ...s, paidOnly: true })); + expect(select({ segments: paid })).toEqual({ + clips: [], + hookSegmentId: null, + notes: ["no usable segments"], + }); + }); + + it("is deterministic", () => { + expect(select({ segments: [...segments].reverse() })).toEqual(select()); + }); +}); + +describe("fitTeaserWeights", () => { + const prior = defaultTeaserWeights("teaser"); + const samples = (count: number) => + Array.from({ length: count }, (_, i) => { + const hook = (i % 10) / 9; + const motion = ((i * 7) % 10) / 9; + return { + features: { speechHook: hook, motion }, + views: 1000, + conversions: Math.round(1000 * (0.01 + 0.2 * hook)), + }; + }); + + it("stays off below the minimum sample count", () => { + expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES - 1), prior)).toBeNull(); + const withoutViews = [ + ...samples(TEASER_LEARNING_MIN_SAMPLES - 1), + { features: {}, views: 0, conversions: 0 }, + ]; + expect(fitTeaserWeights(withoutViews, prior)).toBeNull(); + }); + + it("shifts weight toward features that drive conversion, deterministically", () => { + const learned = fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)!; + expect(learned).not.toBeNull(); + expect(learned.speechHook).toBeGreaterThan(prior.speechHook); + expect(learned.motion).toBeLessThan(prior.motion); + expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)).toEqual(learned); + const more = fitTeaserWeights(samples(200), prior)!; + expect(more.speechHook).toBeGreaterThan(learned.speechHook); + }); +}); + +describe("autoPlanEdl", () => { + const assets = new Map([ + ["asset-a", { durationMs: 30_000, width: 1920, height: 1080, hasSpeech: true }], + ["asset-b", { durationMs: 8000, width: 1080, height: 1920, hasSpeech: false }], + ]); + const context = testContext({ + assets: new Map([ + ["asset-a", { kind: "video", durationMs: 30_000, width: 1920, height: 1080 }], + ["asset-b", { kind: "video", durationMs: 8000, width: 1080, height: 1920 }], + ]), + paidOnly: [{ assetId: "asset-a", startMs: 16_000, endMs: 20_000 }], + templateIds: new Set(["intro-minimal", "end-card-a"]), + goal: "teaser", + }); + + it("plans a valid teaser with hook first, captions, watermark, CTA and redactions", () => { + const kit = testKit({ + intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 }, + endCard: { templateId: "end-card-a", params: { style: "a" }, durationMs: 2000 }, + lutArtifactId: "lut-1", + }); + const edl = autoPlanEdl({ + goal: "teaser", + profile: testProfile(), + brandKit: kit, + segments, + weights: defaultTeaserWeights("teaser"), + assets, + people: [ + { id: "person-1", anonymizeByDefault: "synthetic_identity" }, + { id: "person-2", anonymizeByDefault: "none" }, + ], + marks: [ + { id: "mark-1", treatment: "inpaint", assetId: "asset-a", personId: null }, + { id: "mark-2", treatment: "blur", assetId: "asset-z", personId: null }, + { id: "mark-3", treatment: "none", assetId: "asset-a", personId: null }, + ], + ctaText: "Full video: link in bio", + }); + const result = validateEdl(edl, { ...context, brandKit: kit }); + expect(result.ok ? [] : result.errors).toEqual([]); + // Short vertical profile: no intro, the hook comes first. + expect(edl.intro).toBeUndefined(); + expect(edl.tracks.video[0]!.segmentId).toBe("s2"); + expect(edl.tracks.video[0]!.reframe).toBe("auto"); + expect(edl.tracks.video[0]!.color?.useBrandLut).toBe(true); + expect(edl.outro).toMatchObject({ + templateId: "end-card-a", + params: { style: "a", cta: "Full video: link in bio" }, + }); + expect(edl.captions).toMatchObject({ mode: "both", presetId: "clean", maxCharsPerLine: 24 }); + expect(edl.overlays.map((o) => o.kind)).toEqual(["watermark"]); + expect(edl.redactions.faces.map((f) => [f.target, f.mode])).toEqual([ + [{ personId: "person-1" }, "synthetic_identity"], + [{ unassignedFaces: true }, "blur"], + ]); + expect(edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "inpaint" }]); + expect(edl.metadata).toMatchObject({ + aiAltered: true, + aiAlterations: ["synthetic_identity", "inpaint"], + }); + }); + + it("uses a CTA text card without an end-card template and keeps the intro on long-form profiles", () => { + const profile = testPaidProfile(); + const kit = testKit({ + intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 }, + watermark: undefined, + }); + const teaser = autoPlanEdl({ + goal: "teaser", + profile, + brandKit: kit, + segments, + weights: defaultTeaserWeights("teaser"), + assets, + people: [], + marks: [], + ctaText: "Watch the full video", + }); + expect(teaser.intro?.templateId).toBe("intro-minimal"); + expect(teaser.overlays).toEqual([ + expect.objectContaining({ kind: "cta_end_card", text: "Watch the full video" }), + ]); + expect(teaser.redactions.faces).toEqual([]); + expect(teaser.metadata.aiAltered).toBe(false); + const result = validateEdl(teaser, { ...context, profile, brandKit: kit }); + expect(result.ok ? [] : result.errors).toEqual([]); + }); + + it("plans a full release in capture order including paid-only material on paid profiles", () => { + const profile = testPaidProfile(); + const edl = autoPlanEdl({ + goal: "full_release", + profile, + brandKit: null, + segments, + weights: defaultTeaserWeights("full_release"), + assets, + people: [], + marks: [], + ctaText: null, + }); + expect(edl.tracks.video.map((c) => c.segmentId)).toEqual([ + "s1", + "s2", + "s3", + "s4", + "s5", + "s6", + "s7", + "s8", + ]); + expect(edl.tracks.video.at(-1)!.reframe).toBe("auto"); + expect(edl.tracks.video[0]!.reframe).toBe("none"); + const result = validateEdl(edl, { ...context, profile, brandKit: null, goal: "full_release" }); + expect(result.ok ? [] : result.errors).toEqual([]); + expect(() => + autoPlanEdl({ + goal: "teaser", + profile, + brandKit: null, + segments: [], + weights: defaultTeaserWeights("teaser"), + assets, + people: [], + marks: [], + ctaText: null, + }), + ).toThrow(/no usable segments/); + }); +}); + +describe("rankThumbnailFrames", () => { + it("returns the best frames spread over the video", () => { + const candidates = [ + { tMs: 0, sharpness: 0.5, face: 0, thirds: 0.5, contrast: 0.5 }, + { tMs: 1000, sharpness: 1, face: 1, thirds: 1, contrast: 1 }, + { tMs: 1200, sharpness: 1, face: 1, thirds: 0.9, contrast: 1 }, + { tMs: 5000, sharpness: 0.8, face: 0.5, thirds: 0.5, contrast: 0.5 }, + { tMs: 9000, sharpness: 0.7, face: 0.5, thirds: 0.5, contrast: 0.5 }, + ]; + expect(rankThumbnailFrames(candidates, 3)).toEqual([1000, 5000, 9000]); + expect(rankThumbnailFrames(candidates.slice(1, 3), 3)).toEqual([1000, 1200]); + expect(rankThumbnailFrames(candidates, 0)).toEqual([]); + }); +}); diff --git a/packages/media/src/test-fixtures.ts b/packages/media/src/test-fixtures.ts new file mode 100644 index 0000000..105843f --- /dev/null +++ b/packages/media/src/test-fixtures.ts @@ -0,0 +1,135 @@ +import type { BrandKit, Edl, PlatformProfile } from "@rakazo/contracts"; +import { BrandKitSchema, EdlSchema, PlatformProfileSchema } from "@rakazo/contracts"; +import type { EdlContext } from "./edl.js"; + +/* Synthetic fixtures shared by the unit tests. No real people, places, or platforms. */ + +export function testProfile(overrides: Record = {}): PlatformProfile { + return PlatformProfileSchema.parse({ + id: "example-short-vertical", + name: "Example short vertical", + kind: "free_social", + reviewedAt: "2026-09-01", + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 500, + }, + duration: { hardMinMs: 1000, hardMaxMs: 60_000, recommended: [7000, 15_000, 30_000] }, + safeZones: { topPct: 10, bottomPct: 20, leftPct: 5, rightPct: 12 }, + captions: { recommended: true, maxLines: 2 }, + audio: {}, + text: { + titleMaxChars: 100, + captionMaxChars: 150, + hashtagLimit: 3, + linkPolicy: "bio_only", + bannedPatterns: [{ pattern: "free\\s+money", reason: "misleading claim" }], + }, + content: { thresholds: { explicit_nudity: 0.2, violence: 0.5 } }, + disclosure: { aiAlteredLabelRequired: true }, + consent: { requirePerformerConsent: false, requireAgeVerification: false }, + export: {}, + ...overrides, + }); +} + +export function testPaidProfile(): PlatformProfile { + return testProfile({ + id: "example-paid-video", + name: "Example paid video", + kind: "paid_video", + video: { + aspectRatios: ["16:9"], + width: 1920, + height: 1080, + fps: 30, + codec: "h264", + maxBitrateKbps: 12_000, + maxFileSizeMb: 4000, + }, + duration: { hardMinMs: 1000, hardMaxMs: 3_600_000, recommended: [] }, + safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 }, + text: { captionMaxChars: 2000, hashtagLimit: 10, linkPolicy: "allowed" }, + consent: { requirePerformerConsent: true, requireAgeVerification: true }, + disclosure: { aiAlteredLabelRequired: true, ageRestrictedFlag: true }, + }); +} + +export function testKit(overrides: Partial = {}): BrandKit { + return BrandKitSchema.parse({ + name: "Studio Example", + colors: { + primary: "#112233", + secondary: "#445566", + accent: "#FFAA00", + text: "#FFFFFF", + background: "#000000", + }, + fonts: [{ family: "Example Sans", artifactId: "font-1", role: "caption", license: "OFL" }], + watermark: { artifactId: "logo-1", position: "top_right", opacity: 0.6, scale: 0.15 }, + captionPresets: [ + { + id: "clean", + name: "Clean", + template: "clean", + fontFamily: "Example Sans", + sizePct: 5, + textColor: "#FFFFFF", + }, + ], + musicBeds: [{ artifactId: "music-1", title: "Bed", license: "own", commercialUse: true }], + ...overrides, + }); +} + +/** Two 16:9 clips of a 10 s asset, 4 s total, on a vertical profile. */ +export function testEdl(overrides: Record = {}): Edl { + return EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: "example-short-vertical", + width: 1080, + height: 1920, + fps: 30, + durationTargetMs: 4000, + durationHardMinMs: 1000, + durationHardMaxMs: 60_000, + }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 2000, reframe: "auto" }, + ], + }, + metadata: {}, + ...overrides, + }); +} + +export function testContext(overrides: Partial = {}): EdlContext { + return { + profile: testProfile(), + brandKit: testKit(), + assets: new Map([ + ["asset-a", { kind: "video", durationMs: 10_000, width: 1920, height: 1080 }], + ["asset-b", { kind: "video", durationMs: 8000, width: 1920, height: 1080 }], + ["photo-1", { kind: "photo", durationMs: null, width: 4000, height: 3000 }], + ]), + artifactIds: new Set(["music-1", "music-2", "vo-1"]), + music: new Map([ + ["music-1", { commercialUse: true }], + ["music-2", { commercialUse: false }], + ]), + markIds: new Set(["mark-1"]), + personIds: new Set(["person-1"]), + faceTrackIds: new Set(["track-1"]), + templateIds: new Set(["intro-minimal", "outro-simple", "lower-third-a"]), + paidOnly: [], + ...overrides, + }; +} From 475710e27e6889c14dc3371ffa8a8468a22662fd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:10:02 +0000 Subject: [PATCH 14/16] feat(media): shared service deps and consent-gated model client Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- packages/adapters/src/index.ts | 1 + packages/adapters/src/media-context.test.ts | 82 +++++++++ packages/adapters/src/media-context.ts | 186 ++++++++++++++++++++ 3 files changed, 269 insertions(+) create mode 100644 packages/adapters/src/media-context.test.ts create mode 100644 packages/adapters/src/media-context.ts diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 3848b49..f6a9368 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -76,6 +76,7 @@ export * from "./mcp-emulator.js"; export * from "./mcp-oauth.js"; export * from "./mcp-server-tool.js"; export * from "./mcp-transport.js"; +export * from "./media-context.js"; export * from "./media-emulator.js"; export * from "./media-emulator-clips.js"; export * from "./media-emulator-source.js"; diff --git a/packages/adapters/src/media-context.test.ts b/packages/adapters/src/media-context.test.ts new file mode 100644 index 0000000..37eec33 --- /dev/null +++ b/packages/adapters/src/media-context.test.ts @@ -0,0 +1,82 @@ +import type { AgentRuntime, AgentRuntimeEvent } from "@rakazo/adapter-kit"; +import type { PrismaClient } from "@rakazo/db"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; +import { createMediaModelClient, extractJson } from "./media-context.js"; + +function runtime(reply: string, compaction = true): AgentRuntime { + return { + describe: () => ({ capabilities: { compaction } }), + run: vi.fn(async function* (): AsyncGenerator { + yield { type: "done", text: reply }; + }), + } as unknown as AgentRuntime; +} + +function prisma(consented: boolean): PrismaClient { + return { + aiDataConsent: { findUnique: vi.fn(async () => (consented ? { userId: "u" } : null)) }, + } as unknown as PrismaClient; +} + +const model = async () => ({ provider: "openai", id: "gpt-test" }); +const input = { + spaceId: "s", + userId: "u", + instructions: "Reply with JSON.", + prompt: "Label", + schema: z.object({ title: z.string() }), +}; + +describe("extractJson", () => { + it("reads fenced, wrapped, and bare JSON", () => { + expect(extractJson('```json\n{"a":1}\n```')).toEqual({ a: 1 }); + expect(extractJson('Thinking... {"a":[1,2]} done')).toEqual({ a: [1, 2] }); + expect(extractJson("[1]")).toEqual([1]); + expect(extractJson("no json")).toBeUndefined(); + expect(extractJson("{broken")).toBeUndefined(); + }); +}); + +describe("createMediaModelClient", () => { + it("returns parsed output only with consent for the resolved recipient", async () => { + const withConsent = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"Beach"}'), + resolveModel: model, + }); + expect(await withConsent.generate(input)).toEqual({ title: "Beach" }); + + const run = runtime('{"title":"Beach"}'); + const withoutConsent = createMediaModelClient({ + prisma: prisma(false), + runtime: run, + resolveModel: model, + }); + expect(await withoutConsent.generate(input)).toBeNull(); + expect(run.run).not.toHaveBeenCalled(); + }); + + it("falls back to null for scripted runtimes, missing models, and invalid replies", async () => { + const scripted = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"x"}', false), + resolveModel: model, + }); + expect(await scripted.generate(input)).toBeNull(); + const noModel = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"x"}'), + resolveModel: async () => { + throw new Error("no model"); + }, + }); + expect(await noModel.generate(input)).toBeNull(); + const invalid = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"name":"x"}'), + resolveModel: model, + }); + expect(await invalid.generate(input)).toBeNull(); + }); +}); diff --git a/packages/adapters/src/media-context.ts b/packages/adapters/src/media-context.ts new file mode 100644 index 0000000..78f39cd --- /dev/null +++ b/packages/adapters/src/media-context.ts @@ -0,0 +1,186 @@ +import type { + AgentInputImage, + AgentRunModel, + AgentRuntime, + ArtifactStore, + JobPublisher, + LinkShortener, + MediaInferenceProvider, + MediaPublisher, + MediaRenderer, + MediaSourceProvider, + MotionGraphicsRenderer, +} from "@rakazo/adapter-kit"; +import type { ModelTier } from "@rakazo/contracts"; +import type { MediaRepos, PrismaClient } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import type { ProfileSet } from "@rakazo/media"; +import type { z } from "zod"; +import { aiRecipient } from "./ai-consent.js"; + +/* + * Shared wiring for Media Studio services. Composition roots (API, worker) + * build one MediaServiceDeps; every media service, job handler, and agent tool + * receives it instead of reaching for globals. + */ + +/** Realtime notifications the web and mobile clients subscribe to per space. */ +export type MediaRealtimeEvent = + | { type: "media.source"; sourceId: string; status: string; error?: string | null } + | { type: "media.asset"; assetId: string; ingestState: string } + | { type: "media.collections"; kind: "session" | "theme" } + | { type: "media.people" } + | { type: "media.render"; renderId: string; projectId: string; status: string; progress: number } + | { type: "media.publication"; publicationId: string; status: string }; + +/** + * Structured model calls on behalf of a user in a space. Returns null instead of + * calling a model when no model is configured, the runtime is scripted, or the + * user has not consented to sharing data with the resolved model recipient. + * Callers must then fall back to deterministic behavior. + */ +export interface MediaModelClient { + generate(input: { + spaceId: string; + userId: string; + /** Short system instructions; the reply must be one JSON document. */ + instructions: string; + prompt: string; + images?: AgentInputImage[]; + schema: z.ZodType; + tier?: ModelTier; + signal?: AbortSignal; + }): Promise; +} + +export interface MediaServiceDeps { + prisma: PrismaClient; + repos: MediaRepos; + jobs: JobPublisher; + /** Null when no sidecar is configured; media then stays hidden in the UI. */ + inference: MediaInferenceProvider | null; + renderer: MediaRenderer; + motion: MotionGraphicsRenderer; + publisher: MediaPublisher; + shortener: LinkShortener | null; + sources: Partial>; + /** Absolute DATA_DIR/media. Every stored media path is relative to it. */ + mediaRoot: string; + /** Builtin plus private platform profiles, cached by the composition root. */ + profiles: () => Promise; + artifacts: ArtifactStore; + model: MediaModelClient; + publish: (spaceId: string, event: MediaRealtimeEvent) => Promise; + now: () => Date; +} + +/** Realtime topic for a space's media events. */ +export function mediaTopic(spaceId: string): string { + return `media:${spaceId}`; +} + +const MODEL_TIMEOUT_MS = 120_000; +const RUNTIME_FAILURE = /^(?:I hit a problem:|Unknown model )/i; + +/** Extract the first JSON object or array from a model reply (models may wrap or think aloud). */ +export function extractJson(text: string): unknown { + const trimmed = text.trim(); + const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(trimmed); + const body = fenced?.[1] ?? trimmed; + const start = body.search(/[[{]/); + if (start < 0) return undefined; + const open = body[start]; + const close = open === "{" ? "}" : "]"; + const end = body.lastIndexOf(close); + if (end <= start) return undefined; + try { + return JSON.parse(body.slice(start, end + 1)); + } catch { + return undefined; + } +} + +export function createMediaModelClient(deps: { + prisma: PrismaClient; + runtime: AgentRuntime; + resolveModel: (scope: { + userId: string; + spaceId: string; + tier?: ModelTier; + }) => Promise; +}): MediaModelClient { + return { + async generate(input) { + if (!deps.runtime.describe().capabilities.compaction) return null; + let model: AgentRunModel; + try { + model = await deps.resolveModel({ + userId: input.userId, + spaceId: input.spaceId, + tier: input.tier ?? "fast", + }); + } catch { + return null; + } + if (model.provider === "scripted") return null; + const recipient = aiRecipient({ + provider: model.provider, + use: "model", + modelId: model.id, + baseUrl: model.baseUrl, + }); + if (!recipient) return null; + const consent = await deps.prisma.aiDataConsent.findUnique({ + where: { + userId_spaceId_recipientKey: { + userId: input.userId, + spaceId: input.spaceId, + recipientKey: recipient.key, + }, + }, + }); + if (!consent) return null; + const images = + input.images && model.acceptsImages !== false + ? input.images.slice(0, model.maxImagesPerPrompt ?? input.images.length) + : undefined; + const runId = `media-${crypto.randomUUID()}`; + const signal = input.signal + ? AbortSignal.any([input.signal, AbortSignal.timeout(MODEL_TIMEOUT_MS)]) + : AbortSignal.timeout(MODEL_TIMEOUT_MS); + let reply = ""; + try { + for await (const event of deps.runtime.run( + { + botId: "media-studio", + threadId: runId, + runId, + prompt: input.prompt, + instructions: input.instructions, + history: [], + tools: [], + model, + ...(images && images.length > 0 ? { currentTurnImages: images } : {}), + }, + { + operationId: runId, + traceId: runId, + spaceId: input.spaceId, + userId: input.userId, + signal, + }, + )) { + if (event.type === "done" && event.text) reply = event.text.trim(); + } + } catch (error) { + getLogger().warn("media model call failed", { + reason: error instanceof Error ? error.message : String(error), + }); + return null; + } + if (!reply || RUNTIME_FAILURE.test(reply)) return null; + const parsed = input.schema.safeParse(extractJson(reply)); + return parsed.success ? parsed.data : null; + }, + }; +} From 2556ed197e57c6e3109d930d23e61119f91f8aef Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:11:04 +0000 Subject: [PATCH 15/16] feat(media): service contracts between media slices Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- packages/adapters/src/media-context.ts | 173 ++++++++++++++++++++++++- 1 file changed, 171 insertions(+), 2 deletions(-) diff --git a/packages/adapters/src/media-context.ts b/packages/adapters/src/media-context.ts index 78f39cd..362a854 100644 --- a/packages/adapters/src/media-context.ts +++ b/packages/adapters/src/media-context.ts @@ -11,10 +11,30 @@ import type { MediaSourceProvider, MotionGraphicsRenderer, } from "@rakazo/adapter-kit"; -import type { ModelTier } from "@rakazo/contracts"; +import type { + AiAlteration, + ClientMediaItem, + CreateMarkInput, + Edl, + EdlError, + FfprobeReport, + GateReport, + IntroVariant, + MarketingCopy, + MarkTreatment, + MediaEditPlan, + MediaMark, + MediaPublication, + MediaRender, + MediaRenderOutput, + MediaRenderQuality, + ModelTier, + PlatformProfile, + Utm, +} from "@rakazo/contracts"; import type { MediaRepos, PrismaClient } from "@rakazo/db"; import { getLogger } from "@rakazo/logging"; -import type { ProfileSet } from "@rakazo/media"; +import type { CompileSource, ProfileSet } from "@rakazo/media"; import type { z } from "zod"; import { aiRecipient } from "./ai-consent.js"; @@ -184,3 +204,152 @@ export function createMediaModelClient(deps: { }, }; } + +// --------------------------------------------------------------------------- +// Service contracts between media slices. Each slice implements its part; the +// API, worker, and agent tools only depend on these interfaces. +// --------------------------------------------------------------------------- + +export type CreateMarkInputType = z.infer; + +export interface MediaActor { + spaceId: string; + userId: string; +} + +export interface MediaIngestService { + /** Client-pushed metadata (Apple Photos, iOS): upsert, apply deletions, enqueue work. */ + pushItems( + actor: MediaActor, + input: { + sourceId: string; + items: ClientMediaItem[]; + deletedExternalIds: string[]; + syncToken: string | null; + complete: boolean; + }, + ): Promise<{ created: number; updated: number; unchanged: number; deleted: number }>; + /** Enqueue an incremental sync of a server-side source. */ + requestSync(actor: MediaActor, sourceId: string): Promise; + /** An upload finished: move the file into place and enqueue proxy and analysis. */ + uploadCompleted( + actor: MediaActor, + input: { assetId: string; tempPath: string; sha256: string; fileName: string }, + ): Promise; + recluster(actor: MediaActor): Promise; + /** Remove a source with its assets and every derived file. */ + removeSource(actor: MediaActor, sourceId: string): Promise; +} + +export interface MediaPreparedSources { + sources: Map; + /** Alterations applied while preparing (inpaint, synthetic identity, voice). */ + aiAlterations: AiAlteration[]; + /** Redactions that could not be honored and fell back, for the plan rationale. */ + fallbacks: string[]; +} + +export interface MediaPrivacyService { + /** + * Resolve every clip's source file and apply all face, mark, and region + * redactions from the EDL plus person and mark defaults. Fails closed: a + * redaction that cannot be applied throws instead of returning an unredacted source. + */ + prepareSources(input: { + actor: MediaActor; + projectId: string; + edl: Edl; + quality: MediaRenderQuality; + workDir: string; + signal?: AbortSignal; + }): Promise; + /** Re-detect on the rendered output and evaluate the full gate. */ + gate(input: { + actor: MediaActor; + projectId: string; + edl: Edl; + profile: PlatformProfile; + outputPath: string; + ffprobe: FfprobeReport; + copy: { title: string | null; caption: string | null; tags: string[] }; + bundleHasDisclosure: boolean; + autoFixRound: number; + signal?: AbortSignal; + }): Promise; + /** Create a mark: segment from the prompt, propagate, score, and suggest a fallback. */ + createMark(actor: MediaActor, input: CreateMarkInputType): Promise; + updateMark( + actor: MediaActor, + input: { + markId: string; + treatment?: MarkTreatment; + label?: string | null; + personId?: string | null; + }, + ): Promise; + /** Delete a person, their biometric data, and their files. */ + removePerson(actor: MediaActor, personId: string): Promise; +} + +export interface MediaRenderService { + start( + actor: MediaActor, + input: { planId: string; quality: MediaRenderQuality }, + ): Promise; + cancel(actor: MediaActor, renderId: string): Promise; + /** Absolute path of a finished output for streaming, or null. */ + outputPath(actor: MediaActor, renderId: string, fileName: string): Promise; +} + +export interface MediaMarketingService { + autoPlan( + actor: MediaActor, + input: { projectId: string; profileIds?: string[] }, + ): Promise; + submitPlan( + actor: MediaActor, + input: { + projectId: string; + edl: unknown; + rationale: string | null; + createdBy: "agent" | "user"; + }, + ): Promise<{ ok: true; plan: MediaEditPlan } | { ok: false; errors: EdlError[] }>; + introVariants( + actor: MediaActor, + input: { brandKitId: string; count: number }, + ): Promise; + thumbnails( + actor: MediaActor, + input: { renderId: string; count: number }, + ): Promise; + copy(actor: MediaActor, input: { renderId: string; language: string }): Promise; + createPublication( + actor: MediaActor, + input: { + renderId: string; + destination: string; + targetUrl: string | null; + utm: Utm | null; + title: string | null; + caption: string | null; + tags: string[]; + thumbnailVariant: string | null; + scheduledAt: string | null; + }, + ): Promise; + bundle(actor: MediaActor, publicationId: string): Promise; + cancelPublication(actor: MediaActor, publicationId: string): Promise; + importMetricsCsv( + actor: MediaActor, + input: { mappingId: string; csv: string }, + ): Promise<{ imported: number; skipped: Array<{ row: number; reason: string }> }>; +} + +export interface MediaServices { + deps: MediaServiceDeps; + ingest: MediaIngestService; + privacy: MediaPrivacyService; + render: MediaRenderService; + marketing: MediaMarketingService; +} From 3f9a601dba3d84af1965fe785217ff01caed48a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:24:53 +0000 Subject: [PATCH 16/16] wip(media): w2-marketing partial work (handover, unverified) Stopped mid-implementation for the handover to the project thread. Not typechecked or tested; do not merge as is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- packages/adapters/src/media-marketing-copy.ts | 164 ++++++ .../adapters/src/media-marketing-files.ts | 124 ++++ packages/adapters/src/media-marketing-plan.ts | 532 ++++++++++++++++++ .../src/media-marketing-thumbnails.ts | 350 ++++++++++++ packages/adapters/src/media-metrics.ts | 133 +++++ packages/adapters/src/media-publish.ts | 357 ++++++++++++ packages/adapters/src/media-zip.ts | 143 +++++ 7 files changed, 1803 insertions(+) create mode 100644 packages/adapters/src/media-marketing-copy.ts create mode 100644 packages/adapters/src/media-marketing-files.ts create mode 100644 packages/adapters/src/media-marketing-plan.ts create mode 100644 packages/adapters/src/media-marketing-thumbnails.ts create mode 100644 packages/adapters/src/media-metrics.ts create mode 100644 packages/adapters/src/media-publish.ts create mode 100644 packages/adapters/src/media-zip.ts diff --git a/packages/adapters/src/media-marketing-copy.ts b/packages/adapters/src/media-marketing-copy.ts new file mode 100644 index 0000000..f8ed067 --- /dev/null +++ b/packages/adapters/src/media-marketing-copy.ts @@ -0,0 +1,164 @@ +import type { Edl, MarketingCopy, MediaSegment, PlatformProfile } from "@rakazo/contracts"; +import { MarketingCopySchema } from "@rakazo/contracts"; +import { InvalidMediaInputError, MediaStateConflictError } from "@rakazo/db"; +import { checkCopy, edlDurationMs, isShortForm, normalizeHashtags } from "@rakazo/media"; +import { z } from "zod"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Platform copy for a render (spec 10.2). The model writes from the plan's + * metadata and the transcript of the clips actually used, under the + * profile's limits; it is told to state nothing the material does not show. + * Without a model (or consent) the copy comes from the plan metadata and the + * project title only. Either way hashtags are normalized and capped, long + * fields are cut at a word boundary to the profile limits, and the result is + * checked like the gate's text check; findings are returned as `issues`. + */ + +const TRANSCRIPT_CHARS = 1500; +const CHAPTER_MIN_PROGRAM_MS = 60_000; + +const CopyReplySchema = z.object({ + title: z.string().max(200), + hook: z.string().max(200), + caption: z.string().max(5000), + hashtags: z.array(z.string().max(60)).max(50), + description: z.string().max(5000).optional(), + chapters: z + .array(z.object({ atMs: z.number().int().min(0), title: z.string().max(100) })) + .max(100) + .optional(), + altText: z.string().max(500).optional(), +}); +type CopyReply = z.infer; + +const LANGUAGE = /^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})?$/; + +/** Cut to `max` characters, at the last word boundary when there is one. */ +export function truncateText(value: string, max: number): string { + const chars = [...value.trim()]; + if (chars.length <= max) return value.trim(); + const cut = chars.slice(0, Math.max(0, max)).join(""); + const space = cut.lastIndexOf(" "); + return (space > max * 0.6 ? cut.slice(0, space) : cut).trim(); +} + +function transcriptOf(edl: Edl, segments: MediaSegment[]): string { + const byId = new Map(segments.map((segment) => [segment.id, segment])); + const parts: string[] = []; + for (const clip of [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs)) { + const linked = clip.segmentId ? byId.get(clip.segmentId) : undefined; + const overlapping = linked + ? [linked] + : segments.filter( + (s) => s.assetId === clip.assetId && s.startMs < clip.outMs && s.endMs > clip.inMs, + ); + for (const segment of overlapping) { + const text = segment.transcriptExcerpt?.replace(/\s+/g, " ").trim(); + if (text && !parts.includes(text)) parts.push(text); + } + } + return parts.join(" ").slice(0, TRANSCRIPT_CHARS); +} + +function wantsChapters(profile: PlatformProfile, edl: Edl): boolean { + return !isShortForm(profile) && edlDurationMs(edl) >= CHAPTER_MIN_PROGRAM_MS; +} + +function fallbackCopy(edl: Edl, projectTitle: string, transcript: string): CopyReply { + const title = edl.metadata.title?.trim() || projectTitle; + const sentence = transcript.split(/(?<=[.!?])\s+/)[0]?.trim() ?? ""; + return { + title, + hook: sentence && sentence.length <= 120 ? sentence : title, + caption: edl.metadata.description?.trim() || title, + hashtags: edl.metadata.tags, + ...(edl.metadata.description ? { description: edl.metadata.description } : {}), + }; +} + +function instructions(profile: PlatformProfile, language: string, chapters: boolean): string { + const { text } = profile; + const link = + text.linkPolicy === "allowed" + ? "Links are allowed." + : text.linkPolicy === "bio_only" + ? 'Never put a link in the text; say "link in bio" if a call to action needs one.' + : "Never mention or include links."; + return [ + `You write social video copy in the language "${language}" for ${profile.name}.`, + "Use only facts from the given title, description, tags and transcript. Never invent claims, prices, results, quotes, guarantees or urgency.", + "No clickbait that misrepresents the video, no engagement bait, no promises the video does not keep.", + `Limits: title ${text.titleMaxChars ?? 100} characters, hook 120 characters, caption ${text.captionMaxChars} characters, at most ${text.hashtagLimit} hashtags without the # sign.`, + link, + chapters + ? "Add chapters (atMs from 0, at least 10 s apart) only where the transcript shows topic changes." + : "Do not add chapters.", + "altText describes what is visible for screen readers, factually.", + 'Reply with one JSON object: {"title","hook","caption","hashtags":[],"description","chapters":[{"atMs","title"}],"altText"}.', + ].join("\n"); +} + +export async function generateCopy( + deps: MediaServiceDeps, + actor: MediaActor, + input: { renderId: string; language: string }, +): Promise { + const language = input.language.trim(); + if (!LANGUAGE.test(language)) throw new InvalidMediaInputError("Invalid language code"); + const { spaceId } = actor; + const render = await deps.repos.getRender(spaceId, input.renderId); + const [plan, profileSet, project] = await Promise.all([ + deps.repos.getPlan(spaceId, render.planId), + deps.profiles(), + deps.repos.getProjectRecord(spaceId, render.projectId), + ]); + const profile = profileSet.profiles.get(render.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${render.profileId} is not available`); + const edl = plan.edl; + const segments = await deps.repos.listSegments(spaceId, [ + ...new Set(edl.tracks.video.map((clip) => clip.assetId)), + ]); + const transcript = transcriptOf(edl, segments); + const chapters = wantsChapters(profile, edl); + const durationMs = edlDurationMs(edl); + + const reply = await deps.model.generate({ + spaceId, + userId: actor.userId, + instructions: instructions(profile, language, chapters), + prompt: JSON.stringify({ + projectTitle: project.title, + title: edl.metadata.title ?? null, + description: edl.metadata.description ?? null, + tags: edl.metadata.tags, + durationMs, + transcript: transcript || null, + }), + schema: CopyReplySchema, + }); + const draft = reply ?? fallbackCopy(edl, project.title, transcript); + + const title = truncateText(draft.title, Math.min(200, profile.text.titleMaxChars ?? 200)); + const caption = truncateText(draft.caption, Math.min(5000, profile.text.captionMaxChars)); + const hashtags = normalizeHashtags(draft.hashtags, profile.text.hashtagLimit).map((tag) => + tag.slice(0, 60), + ); + const chapterList = chapters + ? (draft.chapters ?? []) + .filter((chapter) => chapter.atMs < durationMs && chapter.title.trim()) + .sort((a, b) => a.atMs - b.atMs) + : []; + return MarketingCopySchema.parse({ + profileId: profile.id, + language, + title, + hook: truncateText(draft.hook, 200), + caption, + hashtags, + ...(draft.description ? { description: truncateText(draft.description, 5000) } : {}), + ...(chapterList.length > 0 ? { chapters: chapterList } : {}), + ...(draft.altText?.trim() ? { altText: truncateText(draft.altText, 500) } : {}), + issues: checkCopy(profile, { title, caption, tags: hashtags }), + }); +} diff --git a/packages/adapters/src/media-marketing-files.ts b/packages/adapters/src/media-marketing-files.ts new file mode 100644 index 0000000..534c9c2 --- /dev/null +++ b/packages/adapters/src/media-marketing-files.ts @@ -0,0 +1,124 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { AdapterContext } from "@rakazo/adapter-kit"; +import type { MediaRenderOutput } from "@rakazo/contracts"; +import { MediaRenderOutputSchema } from "@rakazo/contracts"; +import { IsolationError } from "@rakazo/db"; +import { resolveInside } from "@rakazo/media/node"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * File and artifact plumbing shared by the marketing services. Media artifacts + * (thumbnails, intro posters, small bundles) are space artifacts without a bot, + * owned by the acting user, like chat attachments. + */ + +/** A stored render output: the public fields plus its path relative to the media root. */ +export type StoredRenderOutput = MediaRenderOutput & { path?: string }; + +export function mediaAdapterContext(actor: MediaActor, operationId: string): AdapterContext { + return { operationId, traceId: operationId, spaceId: actor.spaceId, userId: actor.userId }; +} + +export async function storeMediaArtifact( + deps: Pick, + actor: MediaActor, + input: { name: string; mimeType: string; bytes: Uint8Array }, +): Promise { + const context = mediaAdapterContext(actor, `media-artifact:${input.name}`); + const stored = await deps.artifacts.put(input, context); + const row = await deps.prisma.artifact + .create({ + data: { + spaceId: actor.spaceId, + userId: actor.userId, + name: input.name, + mimeType: input.mimeType, + size: input.bytes.byteLength, + hash: createHash("sha256").update(input.bytes).digest("hex"), + storageKey: stored.id, + }, + select: { id: true }, + }) + .catch(async (error) => { + await deps.artifacts.remove(stored.id, context).catch(() => undefined); + throw error; + }); + return row.id; +} + +/** + * Copy a space artifact (brand font, logo) into `dir` so a local renderer can + * read it. The file name is the artifact id plus its sanitized extension. + */ +export async function materializeArtifact( + deps: Pick, + actor: MediaActor, + artifactId: string, + dir: string, +): Promise { + const row = await deps.prisma.artifact.findFirst({ + where: { id: artifactId, spaceId: actor.spaceId }, + select: { id: true, name: true, storageKey: true }, + }); + if (!row) throw new IsolationError(); + const bytes = await deps.artifacts.get( + row.storageKey, + mediaAdapterContext(actor, `media-artifact-get:${row.id}`), + ); + const ext = path.extname(row.name).toLowerCase(); + const safeId = row.id.replace(/[^A-Za-z0-9_-]/g, ""); + const file = path.join(dir, `${safeId}${/^\.[a-z0-9]{1,5}$/.test(ext) ? ext : ""}`); + await mkdir(dir, { recursive: true }); + await writeFile(file, bytes); + return file; +} + +/** Outputs of a render row, dropping anything that does not parse. */ +export function storedRenderOutputs(outputs: unknown): StoredRenderOutput[] { + if (!Array.isArray(outputs)) return []; + return outputs.flatMap((raw) => { + const parsed = MediaRenderOutputSchema.safeParse(raw); + if (!parsed.success) return []; + const internal = (raw as { path?: unknown }).path; + return [{ ...parsed.data, ...(typeof internal === "string" ? { path: internal } : {}) }]; + }); +} + +export function publicRenderOutput(output: StoredRenderOutput): MediaRenderOutput { + return { + kind: output.kind, + artifactId: output.artifactId, + fileName: output.fileName, + sizeBytes: output.sizeBytes, + }; +} + +/** + * Absolute path of a render output: its stored relative path, or + * `/renders//` when the renderer stored none. + */ +export function renderOutputPath( + deps: Pick, + spaceId: string, + renderId: string, + output: StoredRenderOutput, +): Promise { + return resolveInside( + deps.mediaRoot, + output.path ?? path.join(spaceId, "renders", renderId, path.basename(output.fileName)), + ); +} + +/** Lowercase ASCII slug for UTM values and file names; empty when nothing is left. */ +export function slugify(value: string, max = 100): string { + return value + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, max) + .replace(/-+$/g, ""); +} diff --git a/packages/adapters/src/media-marketing-plan.ts b/packages/adapters/src/media-marketing-plan.ts new file mode 100644 index 0000000..ff9377e --- /dev/null +++ b/packages/adapters/src/media-marketing-plan.ts @@ -0,0 +1,532 @@ +import { mkdir, readFile, rm } from "node:fs/promises"; +import path from "node:path"; +import type { AgentInputImage } from "@rakazo/adapter-kit"; +import type { + BrandKit, + ContentCategory, + Edl, + EdlError, + MediaEditPlan, + MediaProjectGoal, + MediaSegment, + PlatformProfile, + TeaserWeights, +} from "@rakazo/contracts"; +import { TEASER_LEARNING_MIN_SAMPLES } from "@rakazo/contracts"; +import type { MediaProjectRecord, MediaRepos } from "@rakazo/db"; +import { InvalidMediaInputError } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import type { EdlContext, ScoredSegmentInput, SegmentFeatures } from "@rakazo/media"; +import { + autoPlanEdl, + defaultTeaserWeights, + introMs, + scoreSegment, + speechHookScore, + validateEdl, +} from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import { z } from "zod"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Edit plans for a project: the deterministic first cut per profile, an + * optional model ranking of the teaser candidates, and validation of plans the + * agent or user submits. + * + * Model judge (spec 8): only the top JUDGE_TOP_K candidates by deterministic + * score go to the model, and only those whose stored classifier scores already + * pass the target profile thresholds and that show no face of a protected + * person (or an unassigned face next to one). The model receives one still + * frame per candidate, taken from the proxy, plus the transcript excerpt. + * Face crops, embeddings and person data never leave the host. The model's + * order re-weights the candidates: rank r of n scales every feature by + * 1 - r / (2n); every usable segment it did not rank keeps half its weight + * (paid-only segments stay as they are: they only mark the payoff). Without a + * model, consent, or a usable reply the deterministic order stands. + */ + +export const JUDGE_TOP_K = 12; +const JUDGE_FRAME_WIDTH = 320; +const TRANSCRIPT_EXCERPT_CHARS = 300; +const RATIONALE_MAX_CHARS = 2000; + +const RANKING_GOALS: ReadonlySet = new Set([ + "teaser", + "trailer", + "highlight", + "story", +]); + +const JudgeReplySchema = z.object({ + ranking: z + .array(z.object({ segmentId: z.string().min(1).max(100), reason: z.string().max(300) })) + .max(50), +}); + +type AssetRow = Awaited>[number]; +type FaceTrackRow = Awaited>[number]; +type PersonRow = Awaited>[number]; +type ClassifierFrames = Array<{ tMs: number; scores: Partial> }>; + +export interface PlanContext { + project: MediaProjectRecord; + kit: BrandKit | null; + assets: Map; + segments: MediaSegment[]; + faceTracks: FaceTrackRow[]; + /** Every person of the space; plans may reference any of them. */ + people: PersonRow[]; + markIds: Set; + marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; + profiles: Map; +} + +export async function loadPlanContext( + deps: MediaServiceDeps, + actor: MediaActor, + projectId: string, +): Promise { + const { repos } = deps; + const { spaceId } = actor; + const project = await repos.getProjectRecord(spaceId, projectId); + const fromCollections = await repos.collectionAssetIds(spaceId, project.collectionIds); + const assetIds = [...new Set([...project.assetIds, ...fromCollections])]; + const [assetRows, segments, faceTracks, people, marks, kit, profileSet] = await Promise.all([ + repos.listAssetRecords(spaceId, assetIds), + repos.listSegments(spaceId, assetIds), + repos.faceTrackRecords(spaceId, assetIds), + repos.listPersonRecords(spaceId), + repos.listMarks(spaceId), + project.brandKitId ? repos.getBrandKit(spaceId, project.brandKitId) : Promise.resolve(null), + deps.profiles(), + ]); + return { + project, + kit: kit?.kit ?? null, + assets: new Map(assetRows.map((asset) => [asset.id, asset])), + segments, + faceTracks, + people, + markIds: new Set(marks.map((mark) => mark.id)), + marks: marks.map((mark) => ({ + id: mark.id, + treatment: mark.treatment, + assetId: mark.assetId, + personId: mark.personId, + })), + profiles: new Map( + [...profileSet.profiles.entries()].map(([id, entry]) => [id, entry.profile]), + ), + }; +} + +function kitArtifactIds(kit: BrandKit | null): string[] { + if (!kit) return []; + return [ + ...kit.fonts.map((font) => font.artifactId), + ...kit.logoArtifactIds, + ...(kit.lutArtifactId ? [kit.lutArtifactId] : []), + ...(kit.watermark?.artifactId ? [kit.watermark.artifactId] : []), + ...kit.musicBeds.map((bed) => bed.artifactId), + ]; +} + +export function buildEdlContext( + deps: Pick, + ctx: PlanContext, + profile: PlatformProfile, +): EdlContext { + return { + profile, + brandKit: ctx.kit, + assets: new Map( + [...ctx.assets.values()].map((asset) => [ + asset.id, + { + kind: asset.kind as "video" | "photo" | "live_photo", + durationMs: asset.durationMs, + width: asset.width, + height: asset.height, + }, + ]), + ), + artifactIds: new Set(kitArtifactIds(ctx.kit)), + music: new Map( + (ctx.kit?.musicBeds ?? []).map((bed) => [bed.artifactId, { commercialUse: bed.commercialUse }]), + ), + markIds: ctx.markIds, + personIds: new Set(ctx.people.map((person) => person.id)), + faceTrackIds: new Set(ctx.faceTracks.map((track) => track.id)), + templateIds: new Set(deps.motion.templates().map((template) => template.id)), + paidOnly: ctx.segments + .filter((segment) => segment.paidOnly) + .map((segment) => ({ + assetId: segment.assetId, + startMs: segment.startMs, + endMs: segment.endMs, + })), + goal: ctx.project.goal, + }; +} + +function segmentFeatures(segment: MediaSegment): SegmentFeatures { + const features: SegmentFeatures = { ...segment.scores }; + if (features.speechHook === undefined && segment.transcriptExcerpt) { + features.speechHook = speechHookScore(segment.transcriptExcerpt); + } + return features; +} + +function scoredInputs(ctx: PlanContext): ScoredSegmentInput[] { + return ctx.segments.map((segment) => ({ + id: segment.id, + assetId: segment.assetId, + startMs: segment.startMs, + endMs: segment.endMs, + capturedAt: ctx.assets.get(segment.assetId)?.capturedAt ?? null, + features: segmentFeatures(segment), + paidOnly: segment.paidOnly, + transcript: segment.transcriptExcerpt, + })); +} + +/** Scale features so the model's order counts as much as the deterministic score. */ +export function applyRanking( + segments: ScoredSegmentInput[], + order: string[], +): ScoredSegmentInput[] { + const rank = new Map(order.map((id, index) => [id, index])); + return segments.map((segment) => { + // Paid-only segments are never used; they only mark the payoff, so they keep their score. + if (segment.paidOnly) return segment; + const r = rank.get(segment.id); + const factor = r === undefined ? 0.5 : 1 - r / (2 * order.length); + const features = Object.fromEntries( + Object.entries(segment.features).map(([feature, value]) => [feature, (value ?? 0) * factor]), + ) as SegmentFeatures; + return { ...segment, features }; + }); +} + +async function classifierFrames( + deps: MediaServiceDeps, + spaceId: string, + assetIds: Set, +): Promise> { + const analyses = await deps.repos.latestAnalysesByKind(spaceId, "classifier"); + const out = new Map(); + for (const [assetId, row] of analyses) { + if (!assetIds.has(assetId)) continue; + let data: unknown = row.data; + if ((data === null || data === undefined) && row.dataPath) { + try { + data = JSON.parse(await readFile(await resolveInside(deps.mediaRoot, row.dataPath), "utf8")); + } catch { + continue; + } + } + const frames = (data as { frames?: unknown } | null)?.frames; + if (!Array.isArray(frames)) continue; + out.set( + assetId, + frames.filter( + (frame): frame is ClassifierFrames[number] => + typeof frame?.tMs === "number" && typeof frame.scores === "object" && frame.scores !== null, + ), + ); + } + return out; +} + +/** Fail closed: a segment without classifier frames in its range does not pass. */ +function classifierPasses( + frames: ClassifierFrames | undefined, + segment: { startMs: number; endMs: number }, + profile: PlatformProfile, +): boolean { + const inRange = (frames ?? []).filter( + (frame) => frame.tMs >= segment.startMs && frame.tMs <= segment.endMs, + ); + if (inRange.length === 0) return false; + return inRange.every((frame) => + Object.entries(profile.content.thresholds).every(([category, threshold]) => { + const score = frame.scores[category as ContentCategory]; + return score === undefined || threshold === undefined || score <= threshold; + }), + ); +} + +function protectedPersonIds(ctx: PlanContext): Set { + return new Set( + ctx.people.filter((person) => person.anonymizeByDefault !== "none").map((p) => p.id), + ); +} + +function showsProtectedFace( + ctx: PlanContext, + protectedIds: Set, + segment: { assetId: string; startMs: number; endMs: number }, +): boolean { + if (protectedIds.size === 0) return false; + return ctx.faceTracks.some( + (track) => + track.assetId === segment.assetId && + track.startMs < segment.endMs && + track.endMs > segment.startMs && + (track.personId === null || protectedIds.has(track.personId)), + ); +} + +interface Judgement { + order: string[]; + reasons: Map; +} + +async function judgeCandidates( + deps: MediaServiceDeps, + actor: MediaActor, + ctx: PlanContext, + profile: PlatformProfile, + segments: ScoredSegmentInput[], + weights: TeaserWeights, +): Promise { + const top = segments + .filter((segment) => !segment.paidOnly && segment.endMs > segment.startMs) + .map((segment) => ({ segment, score: scoreSegment(segment.features, weights) })) + .sort((a, b) => b.score - a.score || a.segment.id.localeCompare(b.segment.id)) + .slice(0, JUDGE_TOP_K); + if (top.length < 2) return null; + const classifier = await classifierFrames( + deps, + actor.spaceId, + new Set(top.map((c) => c.segment.assetId)), + ); + const protectedIds = protectedPersonIds(ctx); + const eligible = top.filter( + ({ segment }) => + classifierPasses(classifier.get(segment.assetId), segment, profile) && + !showsProtectedFace(ctx, protectedIds, segment), + ); + if (eligible.length < 2) return null; + + const workRelative = path.join(actor.spaceId, "tmp", `judge-${crypto.randomUUID()}`); + const workDir = await resolveInside(deps.mediaRoot, workRelative); + await mkdir(workDir, { recursive: true }); + try { + const images: AgentInputImage[] = []; + const sent: typeof eligible = []; + for (const candidate of eligible) { + const asset = ctx.assets.get(candidate.segment.assetId); + const isVideo = asset?.kind === "video"; + const source = isVideo ? asset?.proxyPath : (asset?.thumbPath ?? asset?.proxyPath); + if (!source) continue; + const outPath = path.join(workDir, `${images.length}.png`); + try { + await deps.renderer.extractFrame({ + path: await resolveInside(deps.mediaRoot, source), + tMs: isVideo + ? Math.round((candidate.segment.startMs + candidate.segment.endMs) / 2) + : 0, + outPath, + width: JUDGE_FRAME_WIDTH, + }); + images.push({ + name: `${candidate.segment.id}.png`, + mimeType: "image/png", + data: new Uint8Array(await readFile(outPath)), + }); + sent.push(candidate); + } catch (error) { + getLogger().warn("media judge frame failed", { + reason: error instanceof Error ? error.message : String(error), + }); + } + } + if (sent.length < 2) return null; + const lines = sent.map(({ segment, score }, index) => { + const transcript = (segment.transcript ?? "").replace(/\s+/g, " ").trim(); + return [ + `${index + 1}. id=${segment.id} image=${segment.id}.png`, + `duration=${segment.endMs - segment.startMs}ms score=${score.toFixed(3)}`, + transcript ? `transcript="${transcript.slice(0, TRANSCRIPT_EXCERPT_CHARS)}"` : "", + ] + .filter(Boolean) + .join(" "); + }); + const reply = await deps.model.generate({ + spaceId: actor.spaceId, + userId: actor.userId, + instructions: [ + "You rank candidate segments for a short promotional video.", + `Goal: ${ctx.project.goal}. Platform: ${profile.name}. The first ${profile.duration.hookWindowMs} ms must hook the viewer.`, + "Judge only what the frames and transcripts show. Prefer clear, engaging openers that keep the payoff hidden.", + 'Reply with one JSON object: {"ranking":[{"segmentId":"...","reason":"..."}]}, best first, using only the given ids.', + ].join("\n"), + prompt: `Candidates:\n${lines.join("\n")}`, + images, + schema: JudgeReplySchema, + }); + if (!reply) return null; + const ids = new Set(sent.map(({ segment }) => segment.id)); + const order: string[] = []; + const reasons = new Map(); + for (const entry of reply.ranking) { + if (!ids.has(entry.segmentId) || reasons.has(entry.segmentId)) continue; + order.push(entry.segmentId); + reasons.set(entry.segmentId, entry.reason.replace(/\s+/g, " ").trim()); + } + return order.length > 0 ? { order, reasons } : null; + } finally { + await rm(workDir, { recursive: true, force: true }); + } +} + +function rationaleFor(input: { + edl: Edl; + profile: PlatformProfile; + ctx: PlanContext; + learned: boolean; + judgement: Judgement | null; +}): string { + const { edl, profile, ctx } = input; + const lines = [ + `Automatic ${ctx.project.goal} plan for ${profile.name}.`, + input.learned + ? "Weights learned from this space's published results." + : `Default ${ctx.project.goal} weights.`, + ]; + const first = [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs)[0]; + if (first) { + lines.push( + `Hook: clip ${first.id}${first.segmentId ? ` (segment ${first.segmentId})` : ""} starts at ${introMs(edl) + first.atMs} ms, inside the ${profile.duration.hookWindowMs} ms hook window.`, + ); + } + if (edl.intro) lines.push("Brand intro opens the video."); + else if (ctx.kit?.intro) lines.push("Brand intro left out so the hook comes first."); + if (ctx.segments.some((segment) => segment.paidOnly)) lines.push("Paid-only segments excluded."); + if (input.judgement) { + const reasons = input.judgement.order + .slice(0, 3) + .map((id) => `${id}: ${input.judgement?.reasons.get(id) ?? ""}`.trim()); + lines.push(`Model ranking applied (${input.judgement.order.length} ranked): ${reasons.join("; ")}`); + } + return lines.join("\n").slice(0, RATIONALE_MAX_CHARS); +} + +function requireProfile(ctx: PlanContext, profileId: string): PlatformProfile { + if (!ctx.project.profileIds.includes(profileId)) { + throw new InvalidMediaInputError(`Profile ${profileId} is not part of this project`); + } + const profile = ctx.profiles.get(profileId); + if (!profile) throw new InvalidMediaInputError(`Unknown platform profile ${profileId}`); + return profile; +} + +/** First cut per profile, validated and stored as a new plan version by the agent. */ +export async function autoPlan( + deps: MediaServiceDeps, + actor: MediaActor, + input: { projectId: string; profileIds?: string[] }, +): Promise { + const ctx = await loadPlanContext(deps, actor, input.projectId); + const profileIds = [...new Set(input.profileIds ?? ctx.project.profileIds)]; + const profiles = profileIds.map((id) => requireProfile(ctx, id)); + const model = await deps.repos.getTeaserModel(actor.spaceId); + const learned = Boolean(model && model.sampleCount >= TEASER_LEARNING_MIN_SAMPLES); + const weights = + learned && model ? model.weights : defaultTeaserWeights(ctx.project.goal); + const base = scoredInputs(ctx); + const assets = new Map( + [...ctx.assets.values()].map((asset) => [ + asset.id, + { + durationMs: asset.durationMs, + width: asset.width, + height: asset.height, + hasSpeech: ctx.segments.some( + (segment) => segment.assetId === asset.id && Boolean(segment.transcriptExcerpt?.trim()), + ), + }, + ]), + ); + const projectPeople = new Set( + ctx.faceTracks.flatMap((track) => (track.personId ? [track.personId] : [])), + ); + const people = ctx.people + .filter((person) => projectPeople.has(person.id)) + .map((person) => ({ id: person.id, anonymizeByDefault: person.anonymizeByDefault })); + const endCardCta = ctx.kit?.endCard?.params.cta; + + const plans: MediaEditPlan[] = []; + for (const profile of profiles) { + const judgement = RANKING_GOALS.has(ctx.project.goal) + ? await judgeCandidates(deps, actor, ctx, profile, base, weights) + : null; + const segments = judgement ? applyRanking(base, judgement.order) : base; + const edl = autoPlanEdl({ + goal: ctx.project.goal, + profile, + brandKit: ctx.kit, + segments, + weights, + assets, + people, + marks: ctx.marks, + ctaText: typeof endCardCta === "string" ? endCardCta : null, + }); + const validation = validateEdl(edl, buildEdlContext(deps, ctx, profile)); + if (!validation.ok) { + throw new InvalidMediaInputError( + `Automatic plan for ${profile.id} is invalid: ${validation.errors + .map((error) => `${error.path} ${error.message}`) + .join("; ")}`, + ); + } + plans.push( + await deps.repos.createPlanVersion(actor.spaceId, { + projectId: ctx.project.id, + profileId: profile.id, + edl: validation.edl, + createdBy: "agent", + rationale: rationaleFor({ edl: validation.edl, profile, ctx, learned, judgement }), + }), + ); + } + return plans; +} + +export async function submitPlan( + deps: MediaServiceDeps, + actor: MediaActor, + input: { projectId: string; edl: unknown; rationale: string | null; createdBy: "agent" | "user" }, +): Promise<{ ok: true; plan: MediaEditPlan } | { ok: false; errors: EdlError[] }> { + const ctx = await loadPlanContext(deps, actor, input.projectId); + const profileId = (input.edl as { output?: { profileId?: unknown } } | null)?.output?.profileId; + const profile = + typeof profileId === "string" && ctx.project.profileIds.includes(profileId) + ? ctx.profiles.get(profileId) + : undefined; + if (!profile) { + return { + ok: false, + errors: [ + { + code: "profile_mismatch", + path: "output.profileId", + message: `plan must target one of the project's profiles: ${ctx.project.profileIds.join(", ")}`, + }, + ], + }; + } + const validation = validateEdl(input.edl, buildEdlContext(deps, ctx, profile)); + if (!validation.ok) return { ok: false, errors: validation.errors }; + const plan = await deps.repos.createPlanVersion(actor.spaceId, { + projectId: ctx.project.id, + profileId: profile.id, + edl: validation.edl, + createdBy: input.createdBy, + rationale: input.rationale, + }); + return { ok: true, plan }; +} diff --git a/packages/adapters/src/media-marketing-thumbnails.ts b/packages/adapters/src/media-marketing-thumbnails.ts new file mode 100644 index 0000000..213e9dd --- /dev/null +++ b/packages/adapters/src/media-marketing-thumbnails.ts @@ -0,0 +1,350 @@ +import { access, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { BrandKit, ContentCategory, Edl, MediaRenderOutput, PlatformProfile } from "@rakazo/contracts"; +import { MediaStateConflictError } from "@rakazo/db"; +import { + clipLengthMs, + escapeFilterText, + escapeFilterValue, + introMs, + locateOutputTime, + mainProgramMs, + rankThumbnailFrames, +} from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import sharp from "sharp"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; +import type { MediaToolPaths } from "./media-folder-source-probe.js"; +import { runMediaTool } from "./media-folder-source-probe.js"; +import type { StoredRenderOutput } from "./media-marketing-files.js"; +import { + materializeArtifact, + publicRenderOutput, + renderOutputPath, + storeMediaArtifact, + storedRenderOutputs, +} from "./media-marketing-files.js"; + +/* + * Thumbnail and poster variants for a finished render (spec 10.1). + * + * Candidates are 24 evenly spaced frames of the main program plus the frame + * just after every cut. Each is decoded small and scored: sharpness is the + * variance of a 4-neighbour Laplacian, contrast the luminance standard + * deviation (both relative to the best candidate), thirds the gradient energy + * near the rule-of-thirds points against the frame average, and face 1 when a + * stored face track of an unprotected person covers the source time. + * rankThumbnailFrames orders them; the best frames are extracted at full size + * as PNG (no EXIF or text chunks) with the plan title drawn in the brand + * colours inside the safe zone. Every variant passes the content thresholds of + * the profile and, when the project protects a person, must show no detectable + * face; failing frames are dropped and the next candidate takes their place. + */ + +export const THUMBNAIL_CANDIDATES = 24; +const MAX_VARIANTS = 10; +const ANALYSIS_WIDTH = 160; +const MAX_THUMB_WIDTH = 1920; +const CUT_OFFSET_MS = 200; +const TITLE_MAX_CHARS = 80; +const DEJAVU_BOLD = [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", + "/usr/share/fonts/dejavu/DejaVuSans-Bold.ttf", + "/usr/share/fonts/TTF/DejaVuSans-Bold.ttf", +]; + +export function thumbnailVariantName(index: number): string { + return `thumb-${String.fromCharCode(97 + index)}`; +} + +export interface FrameMeasure { + /** Variance of the 4-neighbour Laplacian over the grey frame. */ + laplacianVariance: number; + /** Standard deviation of the grey levels. */ + contrast: number; + /** 0..1: gradient energy near the third intersections relative to the frame. */ + thirds: number; +} + +/** Sharpness, contrast and thirds of a packed 8-bit grey frame. */ +export function measureGrayFrame(data: Uint8Array, width: number, height: number): FrameMeasure { + let sum = 0; + let sumSq = 0; + for (let i = 0; i < width * height; i++) { + const value = data[i] ?? 0; + sum += value; + sumSq += value * value; + } + const pixels = Math.max(1, width * height); + const mean = sum / pixels; + const contrast = Math.sqrt(Math.max(0, sumSq / pixels - mean * mean)); + + let lapSum = 0; + let lapSq = 0; + let lapCount = 0; + let energyAll = 0; + let energyThirds = 0; + let thirdsCount = 0; + const radiusX = width / 8; + const radiusY = height / 8; + const points = [1 / 3, 2 / 3].flatMap((fx) => + [1 / 3, 2 / 3].map((fy) => [fx * width, fy * height] as const), + ); + for (let y = 1; y < height - 1; y++) { + for (let x = 1; x < width - 1; x++) { + const at = (dx: number, dy: number) => data[(y + dy) * width + x + dx] ?? 0; + const center = at(0, 0); + const lap = 4 * center - at(-1, 0) - at(1, 0) - at(0, -1) - at(0, 1); + lapSum += lap; + lapSq += lap * lap; + lapCount += 1; + const gradient = Math.abs(at(1, 0) - at(-1, 0)) + Math.abs(at(0, 1) - at(0, -1)); + energyAll += gradient; + if (points.some(([px, py]) => Math.abs(x - px) <= radiusX && Math.abs(y - py) <= radiusY)) { + energyThirds += gradient; + thirdsCount += 1; + } + } + } + const lapMean = lapCount ? lapSum / lapCount : 0; + const laplacianVariance = lapCount ? Math.max(0, lapSq / lapCount - lapMean * lapMean) : 0; + const meanAll = lapCount ? energyAll / lapCount : 0; + const meanThirds = thirdsCount ? energyThirds / thirdsCount : 0; + const thirds = meanAll > 0 ? Math.min(1, meanThirds / meanAll / 2) : 0; + return { laplacianVariance, contrast, thirds }; +} + +/** Candidate output times: evenly spaced plus just after each cut, inside the main program. */ +export function thumbnailCandidateTimes(edl: Edl, durationMs: number): number[] { + const start = introMs(edl); + const end = Math.min(durationMs, start + mainProgramMs(edl)); + if (end <= start) return []; + const span = end - start; + const times = new Set(); + for (let i = 0; i < THUMBNAIL_CANDIDATES; i++) { + times.add(Math.round(start + ((i + 0.5) * span) / THUMBNAIL_CANDIDATES)); + } + for (const clip of edl.tracks.video) { + const t = start + clip.atMs + Math.min(CUT_OFFSET_MS, clipLengthMs(clip) / 2); + if (t > start && t < end) times.add(Math.round(t)); + } + return [...times].sort((a, b) => a - b); +} + +async function existingFile(candidates: string[]): Promise { + for (const file of candidates) { + try { + await access(file); + return file; + } catch { + // try the next location + } + } + return null; +} + +function ffmpegColor(hex: string | undefined, fallback: string): string { + return hex && /^#[0-9a-fA-F]{6}$/.test(hex) ? `0x${hex.slice(1)}` : fallback; +} + +/** drawtext title band inside the top safe zone; every value is escaped. */ +export function titleOverlayFilter(input: { + text: string; + width: number; + height: number; + profile: PlatformProfile; + kit: BrandKit | null; + fontFile: string | null; +}): string { + const text = input.text.replace(/\s+/g, " ").trim().slice(0, TITLE_MAX_CHARS); + const fontSize = Math.max( + 12, + Math.round(Math.min(input.height / 14, (input.width * 0.9) / Math.max(1, text.length * 0.6))), + ); + const y = Math.round(input.height * (input.profile.safeZones.topPct / 100 + 0.04)); + const font = input.fontFile + ? `fontfile=${escapeFilterValue(input.fontFile)}` + : `font=${escapeFilterValue("DejaVu Sans")}`; + return [ + `drawtext=${font}`, + `text=${escapeFilterText(text)}`, + `fontsize=${fontSize}`, + `fontcolor=${ffmpegColor(input.kit?.colors.text, "white")}`, + "box=1", + `boxcolor=${ffmpegColor(input.kit?.colors.background, "black")}@0.6`, + `boxborderw=${Math.max(4, Math.round(fontSize / 3))}`, + "x=(w-text_w)/2", + `y=${y}`, + ].join(":"); +} + +function exceedsThresholds( + frames: Array<{ scores: Partial> }>, + profile: PlatformProfile, +): boolean { + return frames.some((frame) => + Object.entries(profile.content.thresholds).some(([category, threshold]) => { + const score = frame.scores[category as ContentCategory]; + return score !== undefined && threshold !== undefined && score > threshold; + }), + ); +} + +export async function createThumbnails( + deps: MediaServiceDeps, + actor: MediaActor, + input: { renderId: string; count: number }, + tools: MediaToolPaths = {}, +): Promise { + const { spaceId } = actor; + const row = await deps.repos.getRenderRecord(spaceId, input.renderId); + if (row.status !== "done") throw new MediaStateConflictError("Thumbnails need a finished render"); + const outputs = storedRenderOutputs(row.outputs); + const video = outputs.find((output) => output.kind === "video"); + if (!video) throw new MediaStateConflictError("Render has no video output"); + const videoPath = await renderOutputPath(deps, spaceId, row.id, video); + const [plan, profileSet, project] = await Promise.all([ + deps.repos.getPlan(spaceId, row.planId), + deps.profiles(), + deps.repos.getProjectRecord(spaceId, row.projectId), + ]); + const profile = profileSet.profiles.get(row.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${row.profileId} is not available`); + const edl = plan.edl; + const kit = project.brandKitId + ? (await deps.repos.getBrandKit(spaceId, project.brandKitId)).kit + : null; + const stored = row.ffprobe as { durationMs?: unknown } | null; + const durationMs = + typeof stored?.durationMs === "number" + ? stored.durationMs + : (await deps.renderer.probe(videoPath)).durationMs; + + const assetIds = [...new Set(edl.tracks.video.map((clip) => clip.assetId))]; + const tracks = await deps.repos.faceTrackRecords(spaceId, assetIds); + const personIds = [...new Set(tracks.flatMap((t) => (t.personId ? [t.personId] : [])))]; + const people = personIds.length ? await deps.repos.listPersonRecords(spaceId, personIds) : []; + const protectedIds = new Set( + people.filter((person) => person.anonymizeByDefault !== "none").map((person) => person.id), + ); + const protectsFaces = protectedIds.size > 0 || edl.redactions.faces.length > 0; + + const relativeDir = path.join(spaceId, "renders", row.id, "thumbs"); + const dir = await resolveInside(deps.mediaRoot, relativeDir); + const candidateDir = path.join(dir, "candidates"); + await rm(candidateDir, { recursive: true, force: true }); + await mkdir(candidateDir, { recursive: true }); + try { + const measured: Array<{ tMs: number } & FrameMeasure & { face: number }> = []; + for (const tMs of thumbnailCandidateTimes(edl, durationMs)) { + const file = path.join(candidateDir, `${tMs}.png`); + await deps.renderer.extractFrame({ path: videoPath, tMs, outPath: file, width: ANALYSIS_WIDTH }); + const { data, info } = await sharp(file).greyscale().raw().toBuffer({ resolveWithObject: true }); + const at = locateOutputTime(edl, tMs); + const face = + at && + tracks.some( + (track) => + track.assetId === at.assetId && + track.startMs <= at.sourceMs && + track.endMs >= at.sourceMs && + track.personId !== null && + !protectedIds.has(track.personId), + ) + ? 1 + : 0; + measured.push({ tMs, face, ...measureGrayFrame(data, info.width, info.height) }); + } + const maxSharp = Math.max(1e-9, ...measured.map((m) => m.laplacianVariance)); + const maxContrast = Math.max(1e-9, ...measured.map((m) => m.contrast)); + const order = rankThumbnailFrames( + measured.map((m) => ({ + tMs: m.tMs, + sharpness: m.laplacianVariance / maxSharp, + face: m.face, + thirds: m.thirds, + contrast: m.contrast / maxContrast, + })), + measured.length, + ); + + const title = edl.metadata.title?.trim() || null; + let fontFile: string | null = null; + if (title) { + const heading = kit?.fonts.find((font) => font.role === "heading") ?? kit?.fonts[0]; + fontFile = heading + ? await materializeArtifact(deps, actor, heading.artifactId, path.join(dir, "fonts")) + : await existingFile(DEJAVU_BOLD); + } + const width = Math.min(edl.output.width, MAX_THUMB_WIDTH); + const height = Math.round((width * edl.output.height) / edl.output.width / 2) * 2; + const wanted = Math.max(0, Math.min(MAX_VARIANTS, Math.floor(input.count))); + const results: StoredRenderOutput[] = []; + for (const tMs of order) { + if (results.length >= wanted) break; + const name = thumbnailVariantName(results.length); + const out = path.join(dir, `${name}.png`); + const frame = path.join(candidateDir, `full-${tMs}.png`); + await deps.renderer.extractFrame({ path: videoPath, tMs, outPath: frame, width }); + if (title) { + await runMediaTool(tools.ffmpegPath ?? "ffmpeg", [ + "-v", + "error", + "-y", + "-i", + frame, + "-vf", + titleOverlayFilter({ text: title, width, height, profile, kit, fontFile }), + "-frames:v", + "1", + "-map_metadata", + "-1", + "-bitexact", + "-c:v", + "png", + out, + ]); + } else { + await rename(frame, out); + } + if (deps.inference) { + const content = await deps.inference.classifyContent({ + path: out, + sampleFps: 1, + extraTimesMs: [0], + }); + let rejected = exceedsThresholds(content.frames, profile); + if (!rejected && protectsFaces) { + const faces = await deps.inference.detectFaces({ path: out, sampleFps: 1, timesMs: [0] }); + rejected = faces.frames.some((f) => f.faces.length > 0); + } + if (rejected) { + await rm(out, { force: true }); + continue; + } + } + const bytes = await readFile(out); + const artifactId = await storeMediaArtifact(deps, actor, { + name: `${name}.png`, + mimeType: "image/png", + bytes, + }); + results.push({ + kind: "thumbnail", + artifactId, + fileName: `${name}.png`, + sizeBytes: (await stat(out)).size, + path: path.join(relativeDir, `${name}.png`), + }); + } + await deps.repos.updateRender( + spaceId, + row.id, + { outputs: [...outputs.filter((output) => output.kind !== "thumbnail"), ...results] }, + ["done"], + ); + return results.map(publicRenderOutput); + } finally { + await rm(candidateDir, { recursive: true, force: true }); + } +} diff --git a/packages/adapters/src/media-metrics.ts b/packages/adapters/src/media-metrics.ts new file mode 100644 index 0000000..fec5389 --- /dev/null +++ b/packages/adapters/src/media-metrics.ts @@ -0,0 +1,133 @@ +import type { Edl, MediaSegment } from "@rakazo/contracts"; +import { TEASER_FEATURES, TEASER_LEARNING_MIN_SAMPLES } from "@rakazo/contracts"; +import type { NewMediaMetric, TeaserModelRecord } from "@rakazo/db"; +import { InvalidMediaInputError } from "@rakazo/db"; +import type { SegmentFeatures } from "@rakazo/media"; +import { + clipLengthMs, + defaultTeaserWeights, + fitTeaserWeights, + parseMetricsCsv, + speechHookScore, +} from "@rakazo/media"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Results import and teaser learning (spec 8 "Lernen", 10.6). + * + * A CSV import resolves rows through the saved mapping and records them as + * cumulative snapshots: re-importing a report replaces the same + * (publication, date, source) rows, so numbers never double. After every + * import the teaser weights are refit from all publications with views. Each + * publication contributes the clip-length-weighted mean features of the + * segments its plan used, with its latest views and conversions. Below + * TEASER_LEARNING_MIN_SAMPLES publications nothing is stored and plans keep + * the default weights; resetTeaserModel goes back to them. + */ + +export const METRICS_CSV_MAX_CHARS = 5_000_000; + +/** Clip-length-weighted mean of the features of the segments a plan used. */ +export function planFeatures(edl: Edl, segments: MediaSegment[]): SegmentFeatures | null { + const byId = new Map(segments.map((segment) => [segment.id, segment])); + const totals: Record = {}; + let weight = 0; + for (const clip of edl.tracks.video) { + const used = clip.segmentId + ? [byId.get(clip.segmentId)].filter((s): s is MediaSegment => Boolean(s)) + : segments.filter( + (s) => s.assetId === clip.assetId && s.startMs < clip.outMs && s.endMs > clip.inMs, + ); + if (used.length === 0) continue; + const length = clipLengthMs(clip) / used.length; + for (const segment of used) { + const features: SegmentFeatures = { ...segment.scores }; + if (features.speechHook === undefined && segment.transcriptExcerpt) { + features.speechHook = speechHookScore(segment.transcriptExcerpt); + } + for (const feature of TEASER_FEATURES) { + totals[feature] = (totals[feature] ?? 0) + (features[feature] ?? 0) * length; + } + weight += length; + } + } + if (weight <= 0) return null; + return Object.fromEntries( + TEASER_FEATURES.map((feature) => [feature, (totals[feature] ?? 0) / weight]), + ) as SegmentFeatures; +} + +/** Refit and store the space's teaser weights; null while there are too few results. */ +export async function refitTeaserModel( + deps: Pick, + spaceId: string, +): Promise { + const { repos } = deps; + const rows = (await repos.funnel(spaceId)).filter((row) => row.views > 0); + if (rows.length < TEASER_LEARNING_MIN_SAMPLES) return null; + const plans = new Map(); + const samples: Array<{ edl: Edl; views: number; conversions: number }> = []; + for (const row of rows) { + const publication = await repos.getPublication(spaceId, row.publicationId); + const render = await repos.getRender(spaceId, publication.renderId); + let edl = plans.get(render.planId); + if (!edl) { + edl = (await repos.getPlan(spaceId, render.planId)).edl; + plans.set(render.planId, edl); + } + samples.push({ edl, views: row.views, conversions: row.conversions }); + } + const assetIds = [ + ...new Set(samples.flatMap((s) => s.edl.tracks.video.map((clip) => clip.assetId))), + ]; + const segments = await repos.listSegments(spaceId, assetIds); + const usable = samples.flatMap((sample) => { + const features = planFeatures(sample.edl, segments); + return features + ? [{ features, views: sample.views, conversions: Math.min(sample.conversions, sample.views) }] + : []; + }); + const weights = fitTeaserWeights(usable, defaultTeaserWeights("teaser")); + if (!weights) return null; + return repos.putTeaserModel(spaceId, weights, usable.length); +} + +export async function importMetricsCsv( + deps: Pick, + actor: MediaActor, + input: { mappingId: string; csv: string }, +): Promise<{ imported: number; skipped: Array<{ row: number; reason: string }> }> { + if (input.csv.length > METRICS_CSV_MAX_CHARS) { + throw new InvalidMediaInputError("CSV file is too large"); + } + const { spaceId } = actor; + const mapping = await deps.repos.getMetricMapping(spaceId, input.mappingId); + const parsed = parseMetricsCsv(input.csv, mapping); + const keys = await deps.repos.resolvePublicationKeys( + spaceId, + mapping.publicationKey, + parsed.rows.map((row) => row.publicationKey), + ); + const skipped = [...parsed.skipped]; + const metrics: NewMediaMetric[] = []; + for (const row of parsed.rows) { + const publicationId = keys.get(row.publicationKey.trim()); + if (!publicationId) { + skipped.push({ row: row.row, reason: "publication not found" }); + continue; + } + metrics.push({ + publicationId, + observedAt: row.observedAt.toISOString(), + views: row.views, + clicks: row.clicks, + conversions: row.conversions, + revenueCents: row.revenueCents, + currency: row.currency, + source: "csv_import", + }); + } + const imported = await deps.repos.recordMetrics(spaceId, metrics); + if (imported > 0) await refitTeaserModel(deps, spaceId); + return { imported, skipped: skipped.sort((a, b) => a.row - b.row) }; +} diff --git a/packages/adapters/src/media-publish.ts b/packages/adapters/src/media-publish.ts new file mode 100644 index 0000000..76dafac --- /dev/null +++ b/packages/adapters/src/media-publish.ts @@ -0,0 +1,357 @@ +import { copyFile, link, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { MediaJobHandlers } from "@rakazo/adapter-kit"; +import { mediaPublishJob } from "@rakazo/adapter-kit"; +import type { MediaPublication, Utm } from "@rakazo/contracts"; +import { ATTACHMENT_MAX_BYTES, UtmSchema } from "@rakazo/contracts"; +import { InvalidMediaInputError, MediaStateConflictError } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import { buildTrackingUrl, normalizeHashtags, uploadChecklist } from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import type { MediaActor, MediaMarketingService, MediaServiceDeps } from "./media-context.js"; +import type { StoredRenderOutput } from "./media-marketing-files.js"; +import { + renderOutputPath, + slugify, + storedRenderOutputs, + storeMediaArtifact, +} from "./media-marketing-files.js"; +import { thumbnailVariantName } from "./media-marketing-thumbnails.js"; +import type { ZipEntry } from "./media-zip.js"; +import { writeStoreZip } from "./media-zip.js"; + +/* + * Publications: tracking links, scheduling, and the export bundle (spec 10.3–10.5). + * + * A scheduled publication is a `media.publish` background job with + * `availableAt` (Graphile scheduling, as routines use), keyed per publication + * so rescheduling replaces it and canceling removes it. + * + * The bundle lives in `/bundles//` and is rebuilt from + * scratch on every call. Every file name is a constant from BUNDLE_FILES (or a + * thumbnail variant name), never user input, and the ZIP writer only accepts + * flat names, so extracting the archive cannot escape its folder. The ZIP + * becomes an artifact when it fits the attachment limit (ATTACHMENT_MAX_BYTES, + * the same limit chat artifacts have); larger bundles stay as `bundle.zip` in + * the folder for the media file route and bundleArtifactId stays null. + */ + +export const BUNDLE_FILES = { + video: "video.mp4", + posterPng: "poster.png", + posterJpg: "poster.jpg", + srt: "subtitles.srt", + vtt: "subtitles.vtt", + caption: "caption.txt", + metadata: "metadata.json", + checklist: "UPLOAD-CHECKLIST.md", + disclosure: "AI-DISCLOSURE.txt", + zip: "bundle.zip", +} as const; + +const MAX_THUMBNAIL_VARIANTS = 10; + +export function bundleRelativeDir(spaceId: string, publicationId: string): string { + return path.join(spaceId, "bundles", publicationId); +} + +async function publishEvent(deps: MediaServiceDeps, spaceId: string, pub: MediaPublication) { + await deps.publish(spaceId, { + type: "media.publication", + publicationId: pub.id, + status: pub.status, + }); +} + +/** "9x16"-style format of an output size. */ +function aspectLabel(width: number, height: number): string { + const gcd = (a: number, b: number): number => (b === 0 ? a : gcd(b, a % b)); + const divisor = gcd(width, height) || 1; + return `${width / divisor}x${height / divisor}`; +} + +export async function createPublication( + deps: MediaServiceDeps, + actor: MediaActor, + input: { + renderId: string; + destination: string; + targetUrl: string | null; + utm: Utm | null; + title: string | null; + caption: string | null; + tags: string[]; + thumbnailVariant: string | null; + scheduledAt: string | null; + }, +): Promise { + const { spaceId } = actor; + const render = await deps.repos.getRender(spaceId, input.renderId); + if (render.status !== "done") { + throw new MediaStateConflictError("Only a finished render can be published"); + } + const [plan, project] = await Promise.all([ + deps.repos.getPlan(spaceId, render.planId), + deps.repos.getProjectRecord(spaceId, render.projectId), + ]); + const scheduledAt = input.scheduledAt ? new Date(input.scheduledAt) : null; + if (scheduledAt && Number.isNaN(scheduledAt.getTime())) { + throw new InvalidMediaInputError("Invalid schedule time"); + } + const utm = UtmSchema.parse( + input.utm ?? { + source: render.profileId, + medium: `video-${aspectLabel(plan.edl.output.width, plan.edl.output.height)}`, + campaign: slugify(project.title) || project.id.slice(0, 100), + content: (input.thumbnailVariant ?? render.id).slice(0, 100), + }, + ); + let trackingUrl: string | null = null; + if (input.targetUrl) { + try { + trackingUrl = buildTrackingUrl(input.targetUrl, utm); + } catch (error) { + throw new InvalidMediaInputError(error instanceof Error ? error.message : "Invalid target"); + } + if (deps.shortener) { + try { + trackingUrl = await deps.shortener.shorten(trackingUrl); + } catch (error) { + getLogger().warn("media link shortener failed; keeping the full tracking link", { + shortener: deps.shortener.id, + reason: error instanceof Error ? error.message : String(error), + }); + } + } + } + const publication = await deps.repos.createPublication(spaceId, actor.userId, { + renderId: render.id, + destination: input.destination.trim() || "bundle", + targetUrl: input.targetUrl, + trackingUrl, + utm, + title: input.title, + caption: input.caption, + tags: input.tags, + thumbnailVariant: input.thumbnailVariant, + scheduledAt, + }); + if (scheduledAt) await deps.jobs.enqueue(mediaPublishJob(publication.id, scheduledAt)); + await publishEvent(deps, spaceId, publication); + return publication; +} + +export async function cancelPublication( + deps: MediaServiceDeps, + actor: MediaActor, + publicationId: string, +): Promise { + const publication = await deps.repos.cancelPublication(actor.spaceId, publicationId); + await deps.jobs.cancel(mediaPublishJob(publicationId).replaceKey as string); + await publishEvent(deps, actor.spaceId, publication); + return publication; +} + +async function placeFile(source: string, target: string): Promise { + try { + await link(source, target); + } catch { + await copyFile(source, target); + } +} + +function disclosureText(alterations: string[]): string { + return [ + `This video contains AI alterations: ${alterations.join(", ")}.`, + "Set the platform's label for AI-generated or altered content when you upload it.", + "", + ].join("\n"); +} + +export async function buildBundle( + deps: MediaServiceDeps, + actor: MediaActor, + publicationId: string, +): Promise { + const { spaceId } = actor; + const pub = await deps.repos.getPublication(spaceId, publicationId); + if (pub.status === "canceled" || pub.status === "published") { + throw new MediaStateConflictError(`Publication is already ${pub.status}`); + } + const row = await deps.repos.getRenderRecord(spaceId, pub.renderId); + const [plan, profileSet] = await Promise.all([ + deps.repos.getPlan(spaceId, row.planId), + deps.profiles(), + ]); + const profile = profileSet.profiles.get(pub.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${pub.profileId} is not available`); + const edl = plan.edl; + const outputs = storedRenderOutputs(row.outputs); + const video = outputs.find((output) => output.kind === "video"); + if (!video) throw new MediaStateConflictError("Render has no video output"); + + const dir = await resolveInside(deps.mediaRoot, bundleRelativeDir(spaceId, pub.id)); + await rm(dir, { recursive: true, force: true }); + await mkdir(dir, { recursive: true }); + const entries: ZipEntry[] = []; + const add = async (name: string, source: string) => { + const target = path.join(dir, name); + await placeFile(source, target); + entries.push({ name, path: target }); + }; + const write = async (name: string, content: string) => { + const target = path.join(dir, name); + await writeFile(target, content, "utf8"); + entries.push({ name, path: target }); + }; + const sourceOf = (output: StoredRenderOutput) => renderOutputPath(deps, spaceId, row.id, output); + + await add(BUNDLE_FILES.video, await sourceOf(video)); + + if (profile.export.poster) { + const variants = new Map( + Array.from({ length: MAX_THUMBNAIL_VARIANTS }, (_, i) => thumbnailVariantName(i)).map( + (name) => [name, outputs.find((o) => o.kind === "thumbnail" && o.fileName === `${name}.png`)], + ), + ); + const chosen = + (pub.thumbnailVariant ? variants.get(pub.thumbnailVariant) : undefined) ?? + outputs.find((o) => o.kind === "poster") ?? + outputs.find((o) => o.kind === "thumbnail"); + if (chosen) { + const jpeg = /\.jpe?g$/i.test(chosen.fileName); + await add(jpeg ? BUNDLE_FILES.posterJpg : BUNDLE_FILES.posterPng, await sourceOf(chosen)); + } + for (const [name, output] of variants) { + if (output) await add(`${name}.png`, await sourceOf(output)); + } + } + let hasSubtitles = false; + if (profile.export.subtitles) { + for (const [kind, name] of [ + ["subtitles_srt", BUNDLE_FILES.srt], + ["subtitles_vtt", BUNDLE_FILES.vtt], + ] as const) { + const output = outputs.find((o) => o.kind === kind); + if (!output) continue; + await add(name, await sourceOf(output)); + hasSubtitles = true; + } + } + const hashtags = normalizeHashtags(pub.tags, profile.text.hashtagLimit); + if (profile.export.captionText) { + const parts = [pub.caption?.trim() ?? "", hashtags.join(" ")]; + if (pub.trackingUrl && profile.text.linkPolicy === "allowed") parts.push(pub.trackingUrl); + await write(BUNDLE_FILES.caption, `${parts.filter(Boolean).join("\n\n")}\n`); + } + const aiAlterations = [...new Set(edl.metadata.aiAlterations)].sort(); + const aiAltered = edl.metadata.aiAltered || aiAlterations.length > 0; + await write( + BUNDLE_FILES.metadata, + `${JSON.stringify( + { + publicationId: pub.id, + renderId: row.id, + profile: { id: profile.id, name: profile.name, kind: profile.kind }, + title: pub.title, + caption: pub.caption, + tags: hashtags, + trackingUrl: pub.trackingUrl, + utm: pub.utm, + thumbnailVariant: pub.thumbnailVariant, + aiAltered, + aiAlterations, + ageRestricted: profile.disclosure.ageRestrictedFlag, + paidPartnershipLabel: profile.disclosure.paidPartnershipLabel, + }, + null, + 2, + )}\n`, + ); + if (profile.export.uploadChecklist) { + await write( + BUNDLE_FILES.checklist, + uploadChecklist({ + profile, + aiAltered, + hasCaptions: + hasSubtitles || edl.captions?.mode === "burned" || edl.captions?.mode === "both", + trackingUrl: pub.trackingUrl, + }), + ); + } + if (aiAltered) { + await write( + BUNDLE_FILES.disclosure, + disclosureText(aiAlterations.length > 0 ? aiAlterations : ["unspecified"]), + ); + } + + const zipPath = path.join(dir, BUNDLE_FILES.zip); + const zipBytes = await writeStoreZip(zipPath, entries); + let artifactId: string | null = null; + if (zipBytes <= ATTACHMENT_MAX_BYTES) { + artifactId = await storeMediaArtifact(deps, actor, { + name: `bundle-${slugify(profile.id, 40) || "export"}.zip`, + mimeType: "application/zip", + bytes: new Uint8Array(await readFile(zipPath)), + }); + } + const bundled = await deps.repos.markPublicationBundled(spaceId, pub.id, artifactId); + await publishEvent(deps, spaceId, bundled); + return bundled; +} + +/** + * Worker handler for `media.publish`: build the bundle at the scheduled time, + * hand it to the configured publisher, and record the outcome. A canceled or + * already published publication is left alone. Failures mark the publication + * failed and rethrow so the job queue retries; a retry rebuilds from scratch. + * + * No notification is sent here: MediaServiceDeps has no notification provider, + * so clients learn about the bundle through the realtime publication event. + */ +export function createMediaPublishHandler( + deps: MediaServiceDeps, + marketing: Pick, +): MediaJobHandlers["media.publish"] { + return async ({ publicationId }) => { + const spaceId = await deps.repos.mediaSpaceOf("publication", publicationId); + if (!spaceId) return; + const record = await deps.repos.getPublicationRecord(spaceId, publicationId); + if (record.status === "canceled" || record.status === "published") return; + const actor = { spaceId, userId: record.createdBy }; + try { + const bundled = await marketing.bundle(actor, publicationId); + const profile = (await deps.profiles()).profiles.get(bundled.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${bundled.profileId} is not available`); + const result = await deps.publisher.publish({ + bundleDir: await resolveInside(deps.mediaRoot, bundleRelativeDir(spaceId, publicationId)), + profile, + publicationId, + title: bundled.title, + caption: bundled.caption, + tags: bundled.tags, + }); + if (result.status === "published") { + const published = await deps.repos.markPublicationPublished( + spaceId, + publicationId, + result.externalUrl, + deps.now(), + ); + await publishEvent(deps, spaceId, published); + } + } catch (error) { + const current = await deps.repos.getPublication(spaceId, publicationId); + if (current.status === "canceled") return; + if (current.status !== "failed" && current.status !== "published") { + const failed = await deps.repos + .markPublicationFailed(spaceId, publicationId) + .catch(() => null); + if (failed) await publishEvent(deps, spaceId, failed); + } + throw error; + } + }; +} diff --git a/packages/adapters/src/media-zip.ts b/packages/adapters/src/media-zip.ts new file mode 100644 index 0000000..6ac9b82 --- /dev/null +++ b/packages/adapters/src/media-zip.ts @@ -0,0 +1,143 @@ +import { createReadStream } from "node:fs"; +import { open, rm, stat } from "node:fs/promises"; +import { crc32 } from "node:zlib"; + +/* + * Minimal store-only (no compression) ZIP writer for export bundles. Videos + * are already compressed, so storing keeps it fast and dependency-free. + * + * Entry names are flat file names from the bundle's fixed allowlist: no + * directories, no "..", no absolute paths, so extracting can never write + * outside the target folder (zip slip). Timestamps are a fixed 1980-01-01 so + * the same inputs give the same archive. No ZIP64: an entry or archive past + * 4 GiB is rejected with ZipTooLargeError. + */ + +const ENTRY_NAME = /^[A-Za-z0-9][A-Za-z0-9_-]*(\.[A-Za-z0-9_-]+)*$/; +const MAX_ENTRY_NAME = 100; +const MAX_ZIP32 = 0xffff_fffe; +const MAX_ENTRIES = 0xffff; +const DOS_TIME = 0; +/** 1980-01-01 in DOS date format (year offset 0, month 1, day 1). */ +const DOS_DATE = (1 << 5) | 1; + +export class ZipTooLargeError extends Error { + constructor() { + super("bundle is larger than a ZIP archive without ZIP64 can hold (4 GiB)"); + this.name = "ZipTooLargeError"; + } +} + +export interface ZipEntry { + /** Flat file name inside the archive. */ + name: string; + /** Absolute path of the file to store. */ + path: string; +} + +/** Whether a name is a safe flat archive entry (letters, digits, "-", "_", inner dots). */ +export function isSafeZipEntryName(name: string): boolean { + return name.length <= MAX_ENTRY_NAME && ENTRY_NAME.test(name); +} + +async function fileCrc(file: string, signal?: AbortSignal): Promise { + let crc = 0; + for await (const chunk of createReadStream(file, { signal })) { + crc = crc32(chunk as Buffer, crc); + } + return crc >>> 0; +} + +/** Write a store-only ZIP of `entries` to `outPath`, replacing it. Returns the archive size. */ +export async function writeStoreZip( + outPath: string, + entries: ZipEntry[], + signal?: AbortSignal, +): Promise { + if (entries.length > MAX_ENTRIES) throw new ZipTooLargeError(); + const seen = new Set(); + for (const entry of entries) { + if (!isSafeZipEntryName(entry.name)) throw new Error(`unsafe zip entry name: ${entry.name}`); + if (seen.has(entry.name)) throw new Error(`duplicate zip entry: ${entry.name}`); + seen.add(entry.name); + } + const handle = await open(outPath, "w", 0o600); + let offset = 0; + const write = async (bytes: Uint8Array) => { + signal?.throwIfAborted(); + await handle.write(bytes); + offset += bytes.byteLength; + }; + const central: Buffer[] = []; + try { + for (const entry of entries) { + const size = (await stat(entry.path)).size; + const name = Buffer.from(entry.name, "ascii"); + if (size > MAX_ZIP32 || offset + 30 + name.length + size > MAX_ZIP32) { + throw new ZipTooLargeError(); + } + const crc = await fileCrc(entry.path, signal); + const headerOffset = offset; + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(10, 4); // version needed: 1.0, stored + local.writeUInt16LE(0, 6); // flags + local.writeUInt16LE(0, 8); // method: store + local.writeUInt16LE(DOS_TIME, 10); + local.writeUInt16LE(DOS_DATE, 12); + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(size, 18); + local.writeUInt32LE(size, 22); + local.writeUInt16LE(name.length, 26); + local.writeUInt16LE(0, 28); + await write(local); + await write(name); + let copied = 0; + for await (const chunk of createReadStream(entry.path, { signal })) { + await write(chunk as Buffer); + copied += (chunk as Buffer).byteLength; + } + if (copied !== size) throw new Error(`${entry.name} changed while archiving`); + + const record = Buffer.alloc(46); + record.writeUInt32LE(0x02014b50, 0); + record.writeUInt16LE((3 << 8) | 20, 4); // made by: Unix, 2.0 + record.writeUInt16LE(10, 6); + record.writeUInt16LE(0, 8); + record.writeUInt16LE(0, 10); + record.writeUInt16LE(DOS_TIME, 12); + record.writeUInt16LE(DOS_DATE, 14); + record.writeUInt32LE(crc, 16); + record.writeUInt32LE(size, 20); + record.writeUInt32LE(size, 24); + record.writeUInt16LE(name.length, 28); + record.writeUInt16LE(0, 30); // extra + record.writeUInt16LE(0, 32); // comment + record.writeUInt16LE(0, 34); // disk + record.writeUInt16LE(0, 36); // internal attributes + record.writeUInt32LE((0o100644 << 16) >>> 0, 38); // regular file, rw-r--r-- + record.writeUInt32LE(headerOffset, 42); + central.push(record, name); + } + const directory = Buffer.concat(central); + const directoryOffset = offset; + if (directoryOffset + directory.length + 22 > MAX_ZIP32) throw new ZipTooLargeError(); + await write(directory); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(0, 4); + end.writeUInt16LE(0, 6); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(directoryOffset, 16); + end.writeUInt16LE(0, 20); + await write(end); + await handle.close(); + return offset; + } catch (error) { + await handle.close().catch(() => undefined); + await rm(outPath, { force: true }); + throw error; + } +}