diff --git a/.env.example b/.env.example index 5d588c8..5f8d9eb 100644 --- a/.env.example +++ b/.env.example @@ -236,3 +236,10 @@ EVOO_COCKPIT_CORS_ORIGIN= # Expo: English or Simplified Chinese (en | zh-CN). Overridden by SecureStore key # rakazo.uiLocale from Account → Language. # EXPO_PUBLIC_DEFAULT_UI_LOCALE=en + +# Optional Media Studio worker (local ffmpeg + ML sidecar, compose profile "media"). +# Without it, Media Studio stays hidden and media jobs are skipped. +# MEDIA_WORKER_URL=http://127.0.0.1:8790 +# MEDIA_WORKER_TOKEN=REPLACE_WITH_RANDOM_TOKEN +# Private platform profiles that override the shipped ones. Keep outside the repo. +# MEDIA_PROFILES_DIR=~/.evoo/media-profiles diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad09384..89a7429 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,15 @@ jobs: - run: pnpm db:generate env: DATABASE_URL: postgres://rakazo:rakazo@127.0.0.1:5433/rakazo + - name: Ensure ffmpeg for media tests + shell: bash + run: | + if command -v ffmpeg >/dev/null && command -v ffprobe >/dev/null; then exit 0; fi + if sudo -n true 2>/dev/null; then + sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg + else + echo "::warning::ffmpeg is missing on this runner; media render tests will be skipped" + fi - run: pnpm test test-integration: diff --git a/apps/api/package.json b/apps/api/package.json index 6fbddbe..f2ef08c 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -19,6 +19,7 @@ "@rakazo/core": "workspace:*", "@rakazo/db": "workspace:*", "@rakazo/logging": "workspace:*", + "@rakazo/media": "workspace:*", "@rakazo/memory": "workspace:*", "hono": "4.13.7" }, diff --git a/apps/api/src/media.ts b/apps/api/src/media.ts new file mode 100644 index 0000000..6b4ec4a --- /dev/null +++ b/apps/api/src/media.ts @@ -0,0 +1,113 @@ +import { implement, ORPCError } from "@orpc/server"; +import type { Actor } from "@rakazo/contracts"; +import { appContract } from "@rakazo/contracts"; +import type { PrismaClient } from "@rakazo/db"; + +export interface MediaRouterDeps { + prisma: PrismaClient; + dataDir: string; +} + +// Phase A scaffold: every procedure is replaced by its real implementation. +function notImplemented(): never { + throw new ORPCError("NOT_IMPLEMENTED", { message: "Media Studio is not available yet" }); +} + +export function createMediaRouter(_deps: MediaRouterDeps) { + const os = implement(appContract.media).$context<{ actor: Actor | null; signal?: AbortSignal }>(); + const authed = os.use(async ({ context, next }) => { + if (!context.actor) throw new ORPCError("UNAUTHORIZED"); + return next({ context: { ...context, actor: context.actor } }); + }); + + return os.router({ + capabilities: authed.capabilities.handler(notImplemented), + sources: { + list: authed.sources.list.handler(notImplemented), + createFolder: authed.sources.createFolder.handler(notImplemented), + registerClient: authed.sources.registerClient.handler(notImplemented), + pushItems: authed.sources.pushItems.handler(notImplemented), + sync: authed.sources.sync.handler(notImplemented), + remove: authed.sources.remove.handler(notImplemented), + }, + assets: { + search: authed.assets.search.handler(notImplemented), + get: authed.assets.get.handler(notImplemented), + setPaidOnly: authed.assets.setPaidOnly.handler(notImplemented), + requestUpload: authed.assets.requestUpload.handler(notImplemented), + }, + uploads: { + begin: authed.uploads.begin.handler(notImplemented), + status: authed.uploads.status.handler(notImplemented), + complete: authed.uploads.complete.handler(notImplemented), + }, + collections: { + list: authed.collections.list.handler(notImplemented), + get: authed.collections.get.handler(notImplemented), + rename: authed.collections.rename.handler(notImplemented), + merge: authed.collections.merge.handler(notImplemented), + split: authed.collections.split.handler(notImplemented), + createManual: authed.collections.createManual.handler(notImplemented), + recluster: authed.collections.recluster.handler(notImplemented), + }, + people: { + list: authed.people.list.handler(notImplemented), + update: authed.people.update.handler(notImplemented), + merge: authed.people.merge.handler(notImplemented), + assignTrack: authed.people.assignTrack.handler(notImplemented), + recordConsent: authed.people.recordConsent.handler(notImplemented), + remove: authed.people.remove.handler(notImplemented), + }, + marks: { + list: authed.marks.list.handler(notImplemented), + create: authed.marks.create.handler(notImplemented), + update: authed.marks.update.handler(notImplemented), + remove: authed.marks.remove.handler(notImplemented), + }, + brandKits: { + list: authed.brandKits.list.handler(notImplemented), + upsert: authed.brandKits.upsert.handler(notImplemented), + remove: authed.brandKits.remove.handler(notImplemented), + templates: authed.brandKits.templates.handler(notImplemented), + introVariants: authed.brandKits.introVariants.handler(notImplemented), + }, + profiles: { + list: authed.profiles.list.handler(notImplemented), + get: authed.profiles.get.handler(notImplemented), + }, + projects: { + list: authed.projects.list.handler(notImplemented), + get: authed.projects.get.handler(notImplemented), + create: authed.projects.create.handler(notImplemented), + update: authed.projects.update.handler(notImplemented), + remove: authed.projects.remove.handler(notImplemented), + autoPlan: authed.projects.autoPlan.handler(notImplemented), + }, + plans: { + submit: authed.plans.submit.handler(notImplemented), + }, + renders: { + start: authed.renders.start.handler(notImplemented), + get: authed.renders.get.handler(notImplemented), + cancel: authed.renders.cancel.handler(notImplemented), + thumbnails: authed.renders.thumbnails.handler(notImplemented), + }, + marketing: { + copy: authed.marketing.copy.handler(notImplemented), + }, + publications: { + list: authed.publications.list.handler(notImplemented), + create: authed.publications.create.handler(notImplemented), + bundle: authed.publications.bundle.handler(notImplemented), + markPublished: authed.publications.markPublished.handler(notImplemented), + cancel: authed.publications.cancel.handler(notImplemented), + }, + metrics: { + mappings: authed.metrics.mappings.handler(notImplemented), + saveMapping: authed.metrics.saveMapping.handler(notImplemented), + importCsv: authed.metrics.importCsv.handler(notImplemented), + record: authed.metrics.record.handler(notImplemented), + funnel: authed.metrics.funnel.handler(notImplemented), + }, + }); +} diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 963a533..b69c2a9 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -166,6 +166,7 @@ import { renameMainConversation, } from "./main-conversation.js"; import { buildMcpUpdateMaterial } from "./mcp-material.js"; +import { createMediaRouter } from "./media.js"; import { disconnectMemoryProvider, getMemoryProviderConfig, @@ -526,6 +527,7 @@ export function createRouter(deps: RouterDeps) { }); return os.router({ + media: createMediaRouter({ prisma: deps.prisma, dataDir: deps.dataDir }), aiConsent: { status: authed.aiConsent.status.handler(({ context, input }) => aiConsentStatus(deps, context.actor, input), diff --git a/apps/desktop/native/photos-helper/.gitignore b/apps/desktop/native/photos-helper/.gitignore new file mode 100644 index 0000000..2d9f16e --- /dev/null +++ b/apps/desktop/native/photos-helper/.gitignore @@ -0,0 +1,2 @@ +.build/ +.swiftpm/ diff --git a/apps/desktop/native/photos-helper/Package.swift b/apps/desktop/native/photos-helper/Package.swift new file mode 100644 index 0000000..ebf6a40 --- /dev/null +++ b/apps/desktop/native/photos-helper/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version:5.9 +import PackageDescription + +// PhotosHelperCore holds everything that does not touch PhotoKit (argument parsing, +// cursors, JSON envelopes, the export path guard, hashing) so XCTest runs without +// Photos access. The executable target is the thin PhotoKit layer. +let package = Package( + name: "photos-helper", + platforms: [.macOS(.v13)], + products: [ + .executable(name: "photos-helper", targets: ["photos-helper"]) + ], + targets: [ + .target(name: "PhotosHelperCore"), + .executableTarget(name: "photos-helper", dependencies: ["PhotosHelperCore"]), + .testTarget(name: "PhotosHelperCoreTests", dependencies: ["PhotosHelperCore"]), + ] +) diff --git a/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Arguments.swift b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Arguments.swift new file mode 100644 index 0000000..43c5c8f --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Arguments.swift @@ -0,0 +1,142 @@ +import Foundation + +public enum ExportVariant: String, Equatable { + /// The untouched original resource, downloaded from iCloud when needed. + case original + /// A smaller rendition of the current version (720p video, 1280 px JPEG). + case proxy +} + +public enum Command: Equatable { + /// `prompt: false` only reads the current status and never shows the system dialog. + case authorize(prompt: Bool) + case list(cursor: ListCursor?, limit: Int, since: Date?) + case asset(localIdentifier: String) + case export(localIdentifier: String, destination: String, variant: ExportVariant) + case changes(token: String) +} + +public let usageText = """ + usage: photos-helper authorize [--no-prompt] + photos-helper list [--cursor ] [--limit <1-1000>] [--since ] + photos-helper asset + photos-helper export --dest [--original|--proxy] + photos-helper changes --token + """ + +/// Parses argv without the program name. Unknown flags, repeated flags and missing +/// values are usage errors so a typo never silently changes behaviour. +public func parseArguments(_ arguments: [String]) throws -> Command { + guard let name = arguments.first else { throw HelperError(.usage, usageText) } + var options = try OptionReader(Array(arguments.dropFirst())) + switch name { + case "authorize": + let noPrompt = options.flag("--no-prompt") + try options.finish(positionals: 0) + return .authorize(prompt: !noPrompt) + case "list": + let cursor = try options.value("--cursor").map(ListCursor.decode) + let limit = try options.value("--limit").map(parseLimit) ?? Limits.defaultPage + let since = try options.value("--since").map { value -> Date in + guard let date = parseTimestamp(value) else { + throw HelperError(.usage, "--since must be an ISO 8601 timestamp.") + } + return date + } + try options.finish(positionals: 0) + return .list(cursor: cursor, limit: limit, since: since) + case "asset": + try options.finish(positionals: 1) + let identifier = try validIdentifier(options.positionals[0]) + return .asset(localIdentifier: identifier) + case "export": + guard let destination = options.value("--dest") else { + throw HelperError(.usage, "export needs --dest .") + } + let original = options.flag("--original") + let proxy = options.flag("--proxy") + if original && proxy { throw HelperError(.usage, "Choose --original or --proxy, not both.") } + try options.finish(positionals: 1) + let identifier = try validIdentifier(options.positionals[0]) + return .export( + localIdentifier: identifier, + destination: destination, + variant: proxy ? .proxy : .original) + case "changes": + guard let token = options.value("--token"), !token.isEmpty else { + throw HelperError(.usage, "changes needs --token .") + } + try options.finish(positionals: 0) + return .changes(token: token) + default: + throw HelperError(.usage, usageText) + } +} + +func parseLimit(_ value: String) throws -> Int { + guard let limit = Int(value), (1...Limits.maxPage).contains(limit) else { + throw HelperError(.usage, "--limit must be between 1 and \(Limits.maxPage).") + } + return limit +} + +func validIdentifier(_ value: String) throws -> String { + guard !value.isEmpty, value.utf16.count <= Limits.externalId, !value.hasPrefix("-"), + !value.unicodeScalars.contains(where: { CharacterSet.controlCharacters.contains($0) }) + else { throw HelperError(.usage, "Invalid local identifier.") } + return value +} + +private struct OptionReader { + private var values: [String: String] = [:] + private var flags: Set = [] + private var consumed: Set = [] + private(set) var positionals: [String] = [] + + private static let valueOptions: Set = ["--cursor", "--limit", "--since", "--dest", "--token"] + private static let flagOptions: Set = ["--no-prompt", "--original", "--proxy"] + + init(_ arguments: [String]) throws { + var index = 0 + while index < arguments.count { + let argument = arguments[index] + if Self.valueOptions.contains(argument) { + guard index + 1 < arguments.count else { + throw HelperError(.usage, "\(argument) needs a value.") + } + guard values[argument] == nil else { throw HelperError(.usage, "\(argument) given twice.") } + values[argument] = arguments[index + 1] + index += 2 + } else if Self.flagOptions.contains(argument) { + guard !flags.contains(argument) else { throw HelperError(.usage, "\(argument) given twice.") } + flags.insert(argument) + index += 1 + } else if argument.hasPrefix("--") { + throw HelperError(.usage, "Unknown option \(argument).") + } else { + positionals.append(argument) + index += 1 + } + } + } + + mutating func value(_ name: String) -> String? { + consumed.insert(name) + return values[name] + } + + mutating func flag(_ name: String) -> Bool { + consumed.insert(name) + return flags.contains(name) + } + + /// Rejects options that belong to another command and surplus positionals. + func finish(positionals expected: Int) throws { + for name in Set(values.keys).union(flags) where !consumed.contains(name) { + throw HelperError(.usage, "\(name) is not valid here.") + } + guard positionals.count == expected else { + throw HelperError(.usage, expected == 0 ? "Unexpected argument." : "Expected \(expected) argument.") + } + } +} diff --git a/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/HelperError.swift b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/HelperError.swift new file mode 100644 index 0000000..6a56ea9 --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/HelperError.swift @@ -0,0 +1,37 @@ +import Foundation + +/// A failure the desktop app maps by `code`. Messages never contain library content +/// beyond the identifier the caller passed in. +public struct HelperError: Error, Equatable { + public enum Code: String, Codable, CaseIterable { + case usage + case notAuthorized = "not_authorized" + case notFound = "not_found" + case pathRejected = "path_rejected" + case destinationExists = "destination_exists" + case network + case exportFailed = "export_failed" + case `internal` + } + + public let code: Code + public let message: String + + public init(_ code: Code, _ message: String) { + self.code = code + self.message = message + } + + /// sysexits(3) style codes so a shell user can tell failure classes apart. + public var exitCode: Int32 { + switch code { + case .usage: return 64 + case .notFound: return 66 + case .pathRejected, .destinationExists: return 73 + case .exportFailed: return 74 + case .network: return 75 + case .notAuthorized: return 77 + case .internal: return 70 + } + } +} diff --git a/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Models.swift b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Models.swift new file mode 100644 index 0000000..d880086 --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Models.swift @@ -0,0 +1,249 @@ +import Foundation + +/// Mirrors `ClientMediaItemSchema` in packages/contracts/src/media.ts. Every key is +/// always present (nullable fields encode as `null`) because the schema has no optional keys. +public struct ClientMediaItem: Encodable, Equatable { + public enum Kind: String, Encodable { + case video + case photo + case livePhoto = "live_photo" + } + + public var externalId: String + public var kind: Kind + public var capturedAt: String? + public var durationMs: Int? + public var width: Int? + public var height: Int? + public var fps: Double? + public var gpsLat: Double? + public var gpsLon: Double? + public var deviceModel: String? + public var albumNames: [String] + public var isFavorite: Bool + public var fileName: String? + + public init( + externalId: String, + kind: Kind, + capturedAt: Date?, + durationSeconds: Double?, + width: Int?, + height: Int?, + latitude: Double?, + longitude: Double?, + albumNames: [String], + isFavorite: Bool, + fileName: String? + ) { + self.externalId = externalId + self.kind = kind + self.capturedAt = capturedAt.map(formatTimestamp) + if let durationSeconds, durationSeconds.isFinite, durationSeconds > 0 { + durationMs = Int((durationSeconds * 1000).rounded()) + } else { + durationMs = nil + } + self.width = width.flatMap { $0 > 0 ? $0 : nil } + self.height = height.flatMap { $0 > 0 ? $0 : nil } + // PhotoKit exposes neither frame rate nor camera model without loading the + // file; the server reads both with ffprobe after upload. + fps = nil + deviceModel = nil + let coordinate = roundedCoordinate(latitude: latitude, longitude: longitude) + gpsLat = coordinate?.latitude + gpsLon = coordinate?.longitude + self.albumNames = Array( + albumNames.map { clip($0, maxCharacters: Limits.albumName) } + .filter { !$0.isEmpty } + .prefix(Limits.albumCount)) + self.isFavorite = isFavorite + self.fileName = fileName.map { clip($0, maxCharacters: Limits.fileName) } + } + + enum CodingKeys: String, CodingKey { + case externalId, kind, capturedAt, durationMs, width, height, fps, gpsLat, gpsLon + case deviceModel, albumNames, isFavorite, fileName + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(externalId, forKey: .externalId) + try container.encode(kind, forKey: .kind) + // `encode` on an Optional writes `null`, unlike the synthesized `encodeIfPresent`. + try container.encode(capturedAt, forKey: .capturedAt) + try container.encode(durationMs, forKey: .durationMs) + try container.encode(width, forKey: .width) + try container.encode(height, forKey: .height) + try container.encode(fps, forKey: .fps) + try container.encode(gpsLat, forKey: .gpsLat) + try container.encode(gpsLon, forKey: .gpsLon) + try container.encode(deviceModel, forKey: .deviceModel) + try container.encode(albumNames, forKey: .albumNames) + try container.encode(isFavorite, forKey: .isFavorite) + try container.encode(fileName, forKey: .fileName) + } +} + +/// Bounds from the contract schema, applied here so one odd asset cannot make a whole +/// page fail validation in the app. +public enum Limits { + public static let externalId = 200 + public static let albumName = 200 + public static let albumCount = 50 + public static let fileName = 255 + public static let defaultPage = 500 + public static let maxPage = 1000 + /// Above this many changed assets a full relisting is cheaper than one huge document. + public static let maxChanges = 5000 +} + +/// Clips to `maxCharacters` UTF-16 units (what zod's `.max()` counts) without splitting +/// a character. +public func clip(_ value: String, maxCharacters: Int) -> String { + if value.utf16.count <= maxCharacters { return value } + var result = "" + var units = 0 + for character in value { + units += character.utf16.count + if units > maxCharacters { break } + result.append(character) + } + return result +} + +/// Two decimals is about 1 km, matching `roundCoordinate` in @rakazo/media. Precise +/// locations never leave the device. +public func roundedCoordinate(latitude: Double?, longitude: Double?) -> ( + latitude: Double, longitude: Double +)? { + guard let latitude, let longitude, latitude.isFinite, longitude.isFinite, + (-90...90).contains(latitude), (-180...180).contains(longitude), + !(latitude == 0 && longitude == 0) + else { return nil } + return ((latitude * 100).rounded() / 100, (longitude * 100).rounded() / 100) +} + +/// RFC 3339 in UTC with milliseconds, accepted by zod's `datetime({ offset: true })`. +public func formatTimestamp(_ date: Date) -> String { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + formatter.timeZone = TimeZone(identifier: "UTC") + return formatter.string(from: date) +} + +public func parseTimestamp(_ value: String) -> Date? { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = formatter.date(from: value) { return date } + formatter.formatOptions = [.withInternetDateTime] + return formatter.date(from: value) +} + +public enum AuthorizationState: String, Encodable { + case authorized, limited, denied, restricted, notDetermined +} + +public struct AuthorizationOutput: Encodable, Equatable { + public let status: AuthorizationState + public init(status: AuthorizationState) { self.status = status } +} + +public struct ListOutput: Encodable { + public let items: [ClientMediaItem] + public let nextCursor: String? + /// Library change token captured before enumeration; `changes --token` resumes from it. + public let changeToken: String? + + public init(items: [ClientMediaItem], nextCursor: String?, changeToken: String?) { + self.items = items + self.nextCursor = nextCursor + self.changeToken = changeToken + } + + enum CodingKeys: String, CodingKey { case items, nextCursor, changeToken } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(items, forKey: .items) + try container.encode(nextCursor, forKey: .nextCursor) + try container.encode(changeToken, forKey: .changeToken) + } +} + +public struct AssetResourceInfo: Encodable, Equatable { + public let type: String + public let fileName: String + public let uti: String + public init(type: String, fileName: String, uti: String) { + self.type = type + self.fileName = fileName + self.uti = uti + } +} + +public struct AssetOutput: Encodable { + public let item: ClientMediaItem + public let modifiedAt: String? + public let hidden: Bool + public let resources: [AssetResourceInfo] + + public init(item: ClientMediaItem, modifiedAt: Date?, hidden: Bool, resources: [AssetResourceInfo]) { + self.item = item + self.modifiedAt = modifiedAt.map(formatTimestamp) + self.hidden = hidden + self.resources = resources + } + + enum CodingKeys: String, CodingKey { case item, modifiedAt, hidden, resources } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(item, forKey: .item) + try container.encode(modifiedAt, forKey: .modifiedAt) + try container.encode(hidden, forKey: .hidden) + try container.encode(resources, forKey: .resources) + } +} + +public struct ChangesOutput: Encodable { + /// Token to store for the next call; null only when a full relisting is required. + public let token: String? + public let changed: [ClientMediaItem] + public let deleted: [String] + public let fullResyncRequired: Bool + + public init(token: String?, changed: [ClientMediaItem], deleted: [String], fullResyncRequired: Bool) { + self.token = token + self.changed = changed + self.deleted = deleted + self.fullResyncRequired = fullResyncRequired + } + + public static func fullResync() -> ChangesOutput { + ChangesOutput(token: nil, changed: [], deleted: [], fullResyncRequired: true) + } + + enum CodingKeys: String, CodingKey { case token, changed, deleted, fullResyncRequired } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(token, forKey: .token) + try container.encode(changed, forKey: .changed) + try container.encode(deleted, forKey: .deleted) + try container.encode(fullResyncRequired, forKey: .fullResyncRequired) + } +} + +public struct ExportOutput: Encodable, Equatable { + public let path: String + public let fileName: String + public let sizeBytes: Int64 + public let sha256: String + public init(path: String, fileName: String, sizeBytes: Int64, sha256: String) { + self.path = path + self.fileName = fileName + self.sizeBytes = sizeBytes + self.sha256 = sha256 + } +} diff --git a/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Output.swift b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Output.swift new file mode 100644 index 0000000..a3f06ed --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Output.swift @@ -0,0 +1,56 @@ +import CryptoKit +import Foundation + +/// stdout carries exactly one JSON document per successful command; stderr carries +/// JSON lines (`{"progress":…}` while exporting, `{"error":…}` on failure). +public func encodeJSON(_ value: T) throws -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + encoder.nonConformingFloatEncodingStrategy = .throw + let data = try encoder.encode(value) + guard let text = String(data: data, encoding: .utf8) else { + throw HelperError(.internal, "Output was not UTF-8.") + } + return text +} + +private struct ErrorEnvelope: Encodable { + struct Body: Encodable { + let code: String + let message: String + } + let error: Body +} + +public func errorLine(_ error: HelperError) -> String { + let envelope = ErrorEnvelope(error: .init(code: error.code.rawValue, message: error.message)) + return (try? encodeJSON(envelope)) + ?? #"{"error":{"code":"internal","message":"Could not encode the error."}}"# +} + +private struct ProgressLine: Encodable { + let progress: Double +} + +/// Progress clamped to 0…1 and rounded to three decimals so the line stays short. +public func progressLine(_ fraction: Double) -> String { + let value = fraction.isFinite ? min(1, max(0, fraction)) : 0 + return (try? encodeJSON(ProgressLine(progress: (value * 1000).rounded() / 1000))) + ?? #"{"progress":0}"# +} + +/// Streams the file through SHA-256 in 1 MiB reads so large videos never load fully. +public func sha256Hex(of url: URL) throws -> (hex: String, sizeBytes: Int64) { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var hasher = SHA256() + var size: Int64 = 0 + while true { + let chunk = try handle.read(upToCount: 1 << 20) ?? Data() + if chunk.isEmpty { break } + hasher.update(data: chunk) + size += Int64(chunk.count) + } + let hex = hasher.finalize().map { String(format: "%02x", $0) }.joined() + return (hex, size) +} diff --git a/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Paging.swift b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Paging.swift new file mode 100644 index 0000000..a8ef97b --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/Paging.swift @@ -0,0 +1,131 @@ +import Foundation + +/// Listing order: undated assets first, then creation date ascending, ties broken by +/// local identifier. PhotoKit can only sort by date, so ties are ordered here. +public struct SortKey: Comparable, Hashable { + /// Milliseconds since 1970; nil for assets without a creation date. + public let creationMillis: Int64? + public let localIdentifier: String + + public init(creationMillis: Int64?, localIdentifier: String) { + self.creationMillis = creationMillis + self.localIdentifier = localIdentifier + } + + public init(creationDate: Date?, localIdentifier: String) { + self.init( + creationMillis: creationDate.map { Int64(($0.timeIntervalSince1970 * 1000).rounded(.down)) }, + localIdentifier: localIdentifier) + } + + public static func < (lhs: SortKey, rhs: SortKey) -> Bool { + if lhs.creationMillis != rhs.creationMillis { + switch (lhs.creationMillis, rhs.creationMillis) { + case (nil, _): return true + case (_, nil): return false + case let (left?, right?): return left < right + } + } + return lhs.localIdentifier < rhs.localIdentifier + } +} + +/// Opaque to the app: base64url of `{"v":1,"d":,"id":""}`. +/// The cursor names the last item returned, so listing resumes after it even when +/// assets were added or removed in between. +public struct ListCursor: Equatable { + public let key: SortKey + + public init(key: SortKey) { self.key = key } + + private struct Payload: Codable { + let v: Int + let d: Int64? + let id: String + + enum CodingKeys: String, CodingKey { case v, d, id } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(v, forKey: .v) + try container.encode(d, forKey: .d) + try container.encode(id, forKey: .id) + } + } + + public func encoded() -> String { + let payload = Payload(v: 1, d: key.creationMillis, id: key.localIdentifier) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + // Encoding a struct of plain values cannot fail. + let data = (try? encoder.encode(payload)) ?? Data() + return base64URLEncode(data) + } + + public static func decode(_ value: String) throws -> ListCursor { + guard value.count <= 2048, let data = base64URLDecode(value), + let payload = try? JSONDecoder().decode(Payload.self, from: data), + payload.v == 1, !payload.id.isEmpty + else { throw HelperError(.usage, "Invalid cursor.") } + return ListCursor(key: SortKey(creationMillis: payload.d, localIdentifier: payload.id)) + } +} + +public func base64URLEncode(_ data: Data) -> String { + data.base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") +} + +public func base64URLDecode(_ value: String) -> Data? { + guard value.allSatisfy({ $0.isASCII && ($0.isLetter || $0.isNumber || $0 == "-" || $0 == "_") }) + else { return nil } + var base64 = value.replacingOccurrences(of: "-", with: "+").replacingOccurrences(of: "_", with: "/") + while base64.count % 4 != 0 { base64.append("=") } + return Data(base64Encoded: base64) +} + +public struct Page { + /// Positions in the source sequence, in listing order. + public let indices: [Int] + public let hasMore: Bool +} + +/// Selects the next page from a sequence sorted by creation date only (ties in any +/// order). Binary search finds the first candidate date; a whole same-date group is +/// collected before sorting so tie order never depends on PhotoKit. +public func nextPage(count: Int, key: (Int) -> SortKey, after cursor: ListCursor?, limit: Int) + -> Page +{ + precondition(limit > 0) + var start = 0 + if let cursor, let cursorMillis = cursor.key.creationMillis { + var low = 0 + var high = count + while low < high { + let mid = (low + high) / 2 + let millis = key(mid).creationMillis + if millis == nil || millis! < cursorMillis { low = mid + 1 } else { high = mid } + } + start = low + } + var candidates: [(index: Int, key: SortKey)] = [] + var stoppedEarly = false + var index = start + while index < count { + let current = key(index) + if candidates.count >= limit, let last = candidates.last, current.creationMillis != last.key.creationMillis { + // Dates ascend, so everything from here on sorts after the collected group. + stoppedEarly = true + break + } + if cursor == nil || cursor!.key < current { + candidates.append((index, current)) + } + index += 1 + } + candidates.sort { $0.key < $1.key } + let hasMore = stoppedEarly || candidates.count > limit + return Page(indices: candidates.prefix(limit).map(\.index), hasMore: hasMore) +} diff --git a/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/PathGuard.swift b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/PathGuard.swift new file mode 100644 index 0000000..cea84c6 --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/PhotosHelperCore/PathGuard.swift @@ -0,0 +1,72 @@ +import Foundation + +/// Where an export is written: a final path and a sibling temporary path inside the +/// destination directory. +public struct ExportDestination: Equatable { + public let directory: URL + public let finalURL: URL + public let temporaryURL: URL + public let fileName: String +} + +/// Resolves the export target. The directory must be absolute, free of `..`, existing +/// and a directory after resolving symlinks (macOS temp dirs live behind /var). The file +/// name comes from Photos, so it is reduced to one safe component and the joined path is +/// checked to stay a direct child of the resolved directory. Existing files are never +/// overwritten. +public func resolveExportDestination( + directory: String, + fileName rawFileName: String, + temporarySuffix: String = UUID().uuidString, + fileManager: FileManager = .default +) throws -> ExportDestination { + guard directory.hasPrefix("/"), !directory.contains("\0") else { + throw HelperError(.pathRejected, "--dest must be an absolute directory path.") + } + let components = directory.split(separator: "/", omittingEmptySubsequences: true) + guard !components.contains(where: { $0 == ".." || $0 == "." }) else { + throw HelperError(.pathRejected, "--dest must not contain . or .. segments.") + } + let resolved = URL(fileURLWithPath: directory, isDirectory: true).resolvingSymlinksInPath() + .standardizedFileURL + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: resolved.path, isDirectory: &isDirectory), isDirectory.boolValue + else { throw HelperError(.pathRejected, "--dest must be an existing directory.") } + + let fileName = sanitizeFileName(rawFileName) + let finalURL = resolved.appendingPathComponent(fileName, isDirectory: false).standardizedFileURL + let temporaryURL = resolved.appendingPathComponent( + ".\(fileName).\(temporarySuffix).partial", isDirectory: false + ).standardizedFileURL + for candidate in [finalURL, temporaryURL] + where candidate.deletingLastPathComponent().path != resolved.path { + throw HelperError(.pathRejected, "Export path left the destination directory.") + } + // lstat semantics: a dangling symlink at the final path also counts as existing. + if (try? fileManager.attributesOfItem(atPath: finalURL.path)) != nil { + throw HelperError(.destinationExists, "The destination file already exists.") + } + return ExportDestination( + directory: resolved, finalURL: finalURL, temporaryURL: temporaryURL, fileName: fileName) +} + +/// One path component with no separators, control characters or leading dots, at most +/// 200 UTF-8 bytes with the extension kept. +public func sanitizeFileName(_ raw: String) -> String { + var cleaned = String( + raw.unicodeScalars.map { scalar -> Character in + if scalar == "/" || scalar == "\\" || scalar == ":" { return "_" } + if CharacterSet.controlCharacters.contains(scalar) { return "_" } + return Character(scalar) + }) + while cleaned.hasPrefix(".") || cleaned.hasPrefix(" ") { cleaned.removeFirst() } + cleaned = cleaned.trimmingCharacters(in: .whitespaces) + if cleaned.isEmpty { return "asset" } + let maxBytes = 200 + if cleaned.utf8.count <= maxBytes { return cleaned } + let ext = (cleaned as NSString).pathExtension + let suffix = ext.isEmpty || ext.utf8.count > 16 ? "" : ".\(ext)" + var stem = suffix.isEmpty ? cleaned : String(cleaned.dropLast(suffix.count)) + while stem.utf8.count + suffix.utf8.count > maxBytes { stem.removeLast() } + return stem + suffix +} diff --git a/apps/desktop/native/photos-helper/Sources/photos-helper/Exporter.swift b/apps/desktop/native/photos-helper/Sources/photos-helper/Exporter.swift new file mode 100644 index 0000000..e491c88 --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/photos-helper/Exporter.swift @@ -0,0 +1,148 @@ +import AVFoundation +import AppKit +import Foundation +import Photos +import PhotosHelperCore + +/// Writes one asset into the destination directory: first to a hidden temporary file, +/// then renamed into place after hashing, so a cancelled export never leaves a file +/// that looks complete. +struct Exporter { + func export( + localIdentifier: String, + directory: String, + variant: ExportVariant, + progress: @escaping (Double) -> Void + ) async throws -> ExportOutput { + guard + let asset = PHAsset.fetchAssets(withLocalIdentifiers: [localIdentifier], options: nil) + .firstObject + else { throw HelperError(.notFound, "No photo or video with that identifier.") } + guard let resource = originalResource(for: asset, variant: variant) else { + throw HelperError(.notFound, "The asset has no exportable resource.") + } + var fileName = resource.originalFilename + if variant == .proxy { + let stem = (fileName as NSString).deletingPathExtension + fileName = stem + (asset.mediaType == .video ? ".mp4" : ".jpg") + } + let destination = try resolveExportDestination(directory: directory, fileName: fileName) + let fileManager = FileManager.default + do { + switch (variant, asset.mediaType) { + case (.original, _): + try await writeResource(resource, to: destination.temporaryURL, progress: progress) + case (.proxy, .video): + try await writeVideoProxy(asset, to: destination.temporaryURL, progress: progress) + case (.proxy, _): + try await writeImageProxy(asset, to: destination.temporaryURL, progress: progress) + } + let digest = try sha256Hex(of: destination.temporaryURL) + // moveItem refuses an existing destination, so nothing is ever overwritten. + try fileManager.moveItem(at: destination.temporaryURL, to: destination.finalURL) + progress(1) + return ExportOutput( + path: destination.finalURL.path, fileName: destination.fileName, + sizeBytes: digest.sizeBytes, sha256: digest.hex) + } catch { + try? fileManager.removeItem(at: destination.temporaryURL) + if let helperError = error as? HelperError { throw helperError } + throw mapExportError(error) + } + } + + /// Original bytes through PHAssetResourceManager; downloads from iCloud when the + /// library keeps only optimized copies on this Mac. + private func writeResource( + _ resource: PHAssetResource, to url: URL, progress: @escaping (Double) -> Void + ) async throws { + let options = PHAssetResourceRequestOptions() + options.isNetworkAccessAllowed = true + options.progressHandler = { value in progress(value) } + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + PHAssetResourceManager.default().writeData(for: resource, toFile: url, options: options) { + error in + if let error { continuation.resume(throwing: error) } else { continuation.resume() } + } + } + } + + /// 720p H.264 MP4 of the current version, without source metadata. + private func writeVideoProxy( + _ asset: PHAsset, to url: URL, progress: @escaping (Double) -> Void + ) async throws { + let options = PHVideoRequestOptions() + options.isNetworkAccessAllowed = true + options.deliveryMode = .highQualityFormat + options.version = .current + options.progressHandler = { value, _, _, _ in progress(value * 0.5) } + let session: AVAssetExportSession = try await withCheckedThrowingContinuation { continuation in + PHImageManager.default().requestExportSession( + forVideo: asset, options: options, exportPreset: AVAssetExportPreset1280x720 + ) { session, info in + if let session { + continuation.resume(returning: session) + } else { + continuation.resume( + throwing: (info?[PHImageErrorKey] as? Error) + ?? HelperError(.exportFailed, "Could not prepare the video export.")) + } + } + } + session.outputURL = url + session.outputFileType = .mp4 + session.shouldOptimizeForNetworkUse = true + session.metadata = [] + session.metadataItemFilter = .forSharing() + await session.export() + guard session.status == .completed else { + throw session.error ?? HelperError(.exportFailed, "The video export did not finish.") + } + } + + /// JPEG of the current version, at most 1280 px on the long edge, without EXIF/GPS. + private func writeImageProxy( + _ asset: PHAsset, to url: URL, progress: @escaping (Double) -> Void + ) async throws { + let options = PHImageRequestOptions() + options.isNetworkAccessAllowed = true + options.deliveryMode = .highQualityFormat + options.resizeMode = .exact + options.version = .current + options.isSynchronous = false + options.progressHandler = { value, _, _, _ in progress(value) } + // highQualityFormat calls the handler exactly once. + let image: NSImage = try await withCheckedThrowingContinuation { continuation in + PHImageManager.default().requestImage( + for: asset, targetSize: CGSize(width: 1280, height: 1280), contentMode: .aspectFit, + options: options + ) { image, info in + if let image { + continuation.resume(returning: image) + } else { + continuation.resume( + throwing: (info?[PHImageErrorKey] as? Error) + ?? HelperError(.exportFailed, "Could not render the photo.")) + } + } + } + guard let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil), + let data = NSBitmapImageRep(cgImage: cgImage).representation( + using: .jpeg, properties: [.compressionFactor: 0.85]) + else { throw HelperError(.exportFailed, "Could not encode the photo.") } + try data.write(to: url, options: .withoutOverwriting) + } +} + +/// iCloud and network failures are retryable; everything else is an export failure. +func mapExportError(_ error: Error) -> HelperError { + if let photosError = error as? PHPhotosError, + photosError.code == .networkAccessRequired || photosError.code == .networkError + { + return HelperError(.network, "The original could not be downloaded from iCloud.") + } + if (error as NSError).domain == NSURLErrorDomain { + return HelperError(.network, "The original could not be downloaded from iCloud.") + } + return HelperError(.exportFailed, "The export failed.") +} diff --git a/apps/desktop/native/photos-helper/Sources/photos-helper/Library.swift b/apps/desktop/native/photos-helper/Sources/photos-helper/Library.swift new file mode 100644 index 0000000..b2a6d8f --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/photos-helper/Library.swift @@ -0,0 +1,190 @@ +import Foundation +import Photos +import PhotosHelperCore + +/// Read-only PhotoKit queries. Nothing here writes to the library. +struct Library { + private let library = PHPhotoLibrary.shared() + + /// Images and videos only; audio and unknown media types are not part of Media Studio. + /// PhotoKit sorts by date only; the store places missing dates first, which is what + /// `SortKey` assumes, and `nextPage` orders ties itself. + private func fetchOptions(since: Date?) -> PHFetchOptions { + let options = PHFetchOptions() + options.includeHiddenAssets = false + var predicates = [ + NSPredicate( + format: "mediaType == %ld || mediaType == %ld", + PHAssetMediaType.image.rawValue, PHAssetMediaType.video.rawValue) + ] + if let since { + predicates.append(NSPredicate(format: "modificationDate >= %@", since as NSDate)) + } + options.predicate = NSCompoundPredicate(andPredicateWithSubpredicates: predicates) + options.sortDescriptors = [NSSortDescriptor(key: "creationDate", ascending: true)] + return options + } + + func list(after cursor: ListCursor?, limit: Int, since: Date?) throws -> ListOutput { + // Captured before enumeration so changes made while listing show up next time. + let changeToken = try encodeToken(library.currentChangeToken) + let result = PHAsset.fetchAssets(with: fetchOptions(since: since)) + let page = nextPage( + count: result.count, + key: { index in + let asset = result.object(at: index) + return SortKey(creationDate: asset.creationDate, localIdentifier: asset.localIdentifier) + }, + after: cursor, + limit: limit) + let assets = page.indices.map { result.object(at: $0) } + let items = assets.compactMap(makeItem(for:)) + let nextCursor = page.hasMore + ? assets.last.map { + ListCursor(key: SortKey(creationDate: $0.creationDate, localIdentifier: $0.localIdentifier)) + .encoded() + } + : nil + return ListOutput(items: items, nextCursor: nextCursor, changeToken: changeToken) + } + + func asset(_ localIdentifier: String) throws -> AssetOutput { + guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [localIdentifier], options: nil).firstObject, + let item = makeItem(for: asset) + else { throw HelperError(.notFound, "No photo or video with that identifier.") } + let resources = PHAssetResource.assetResources(for: asset).map { + AssetResourceInfo( + type: resourceTypeName($0.type), fileName: $0.originalFilename, uti: $0.uniformTypeIdentifier) + } + return AssetOutput( + item: item, modifiedAt: asset.modificationDate, hidden: asset.isHidden, resources: resources) + } + + /// Persistent change history (macOS 13). An expired token or missing history asks the + /// caller for a full relisting instead of guessing. + func changes(since encodedToken: String) throws -> ChangesOutput { + guard let token = decodeToken(encodedToken) else { return .fullResync() } + let current = try encodeToken(library.currentChangeToken) + var changed = Set() + var deleted = Set() + do { + let result = try library.fetchPersistentChanges(since: token) + var detailsError: Error? + result.enumerateChanges { change, stop in + do { + let details = try change.changeDetails(for: .asset) + changed.formUnion(details.insertedLocalIdentifiers) + changed.formUnion(details.updatedLocalIdentifiers) + deleted.formUnion(details.deletedLocalIdentifiers) + if changed.count + deleted.count > Limits.maxChanges { stop.pointee = true } + } catch { + detailsError = error + stop.pointee = true + } + } + if let detailsError { throw detailsError } + } catch let error as PHPhotosError + where error.code == .persistentChangeTokenExpired + || error.code == .persistentChangeDetailsUnavailable + { + return .fullResync() + } catch { + throw HelperError(.internal, "Could not read the Photos change history.") + } + if changed.count + deleted.count > Limits.maxChanges { return .fullResync() } + + changed.subtract(deleted) + var items: [ClientMediaItem] = [] + var found = Set() + if !changed.isEmpty { + PHAsset.fetchAssets(withLocalIdentifiers: Array(changed), options: nil).enumerateObjects { + asset, _, _ in + found.insert(asset.localIdentifier) + if let item = makeItem(for: asset) { items.append(item) } + } + } + // Assets that vanished between the history read and the fetch count as deleted. + deleted.formUnion(changed.subtracting(found)) + items.sort { $0.externalId < $1.externalId } + return ChangesOutput( + token: current, changed: items, deleted: deleted.sorted(), fullResyncRequired: false) + } + + private func encodeToken(_ token: PHPersistentChangeToken) throws -> String { + guard + let data = try? NSKeyedArchiver.archivedData(withRootObject: token, requiringSecureCoding: true) + else { throw HelperError(.internal, "Could not encode the Photos change token.") } + return base64URLEncode(data) + } + + private func decodeToken(_ value: String) -> PHPersistentChangeToken? { + guard let data = base64URLDecode(value) else { return nil } + return try? NSKeyedUnarchiver.unarchivedObject(ofClass: PHPersistentChangeToken.self, from: data) + } +} + +/// Builds the contract item. Returns nil for media types Media Studio does not handle +/// and for identifiers longer than the contract allows. +func makeItem(for asset: PHAsset) -> ClientMediaItem? { + let kind: ClientMediaItem.Kind + switch asset.mediaType { + case .video: kind = .video + case .image: kind = asset.mediaSubtypes.contains(.photoLive) ? .livePhoto : .photo + default: return nil + } + guard asset.localIdentifier.utf16.count <= Limits.externalId else { return nil } + var albums: [String] = [] + PHAssetCollection.fetchAssetCollectionsContaining(asset, with: .album, options: nil) + .enumerateObjects { collection, _, stop in + if let title = collection.localizedTitle { albums.append(title) } + if albums.count >= Limits.albumCount { stop.pointee = true } + } + let coordinate = asset.location?.coordinate + return ClientMediaItem( + externalId: asset.localIdentifier, + kind: kind, + capturedAt: asset.creationDate, + durationSeconds: kind == .video ? asset.duration : nil, + width: asset.pixelWidth, + height: asset.pixelHeight, + latitude: coordinate?.latitude, + longitude: coordinate?.longitude, + albumNames: albums, + isFavorite: asset.isFavorite, + fileName: originalResource(for: asset, variant: .original)?.originalFilename) +} + +/// The resource holding the untouched original: the video track for videos, the still +/// for photos and Live Photos (the paired motion clip is not exported). +func originalResource(for asset: PHAsset, variant: ExportVariant) -> PHAssetResource? { + let resources = PHAssetResource.assetResources(for: asset) + let preferred: [PHAssetResourceType] + switch (asset.mediaType, variant) { + case (.video, .original): preferred = [.video, .fullSizeVideo] + case (.video, .proxy): preferred = [.fullSizeVideo, .video] + case (_, .original): preferred = [.photo, .fullSizePhoto] + case (_, .proxy): preferred = [.fullSizePhoto, .photo] + } + for type in preferred { + if let match = resources.first(where: { $0.type == type }) { return match } + } + return nil +} + +func resourceTypeName(_ type: PHAssetResourceType) -> String { + switch type { + case .photo: return "photo" + case .video: return "video" + case .audio: return "audio" + case .alternatePhoto: return "alternate_photo" + case .fullSizePhoto: return "full_size_photo" + case .fullSizeVideo: return "full_size_video" + case .adjustmentData: return "adjustment_data" + case .adjustmentBasePhoto: return "adjustment_base_photo" + case .pairedVideo: return "paired_video" + case .fullSizePairedVideo: return "full_size_paired_video" + case .adjustmentBasePairedVideo: return "adjustment_base_paired_video" + case .adjustmentBaseVideo: return "adjustment_base_video" + @unknown default: return "other" + } +} diff --git a/apps/desktop/native/photos-helper/Sources/photos-helper/PhotosHelper.swift b/apps/desktop/native/photos-helper/Sources/photos-helper/PhotosHelper.swift new file mode 100644 index 0000000..53f7029 --- /dev/null +++ b/apps/desktop/native/photos-helper/Sources/photos-helper/PhotosHelper.swift @@ -0,0 +1,103 @@ +import Foundation +import Photos +import PhotosHelperCore + +/// Entry point. Exactly one JSON document goes to stdout on success; failures write +/// `{"error":{code,message}}` to stderr and exit non-zero (see HelperError.exitCode). +@main +struct PhotosHelper { + static func main() async { + do { + let command = try parseArguments(Array(CommandLine.arguments.dropFirst())) + let output = try await run(command) + FileHandle.standardOutput.write(Data((output + "\n").utf8)) + exit(0) + } catch let error as HelperError { + fail(error) + } catch { + fail(HelperError(.internal, "Unexpected failure.")) + } + } + + static func fail(_ error: HelperError) -> Never { + FileHandle.standardError.write(Data((errorLine(error) + "\n").utf8)) + exit(error.exitCode) + } + + static func run(_ command: Command) async throws -> String { + switch command { + case .authorize(let prompt): + let status = await authorize(prompt: prompt) + return try encodeJSON(AuthorizationOutput(status: status)) + case .list(let cursor, let limit, let since): + try requireAccess() + return try encodeJSON(try Library().list(after: cursor, limit: limit, since: since)) + case .asset(let localIdentifier): + try requireAccess() + return try encodeJSON(try Library().asset(localIdentifier)) + case .export(let localIdentifier, let destination, let variant): + try requireAccess() + let progress = ProgressReporter() + // Local copies report no progress, so a heartbeat repeats the last value; the app + // treats a silent helper as stalled. + let heartbeat = Task { + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: 15_000_000_000) + if !Task.isCancelled { progress.emit(nil) } + } + } + defer { heartbeat.cancel() } + progress.emit(0) + let result = try await Exporter().export( + localIdentifier: localIdentifier, directory: destination, variant: variant, + progress: { fraction in progress.emit(fraction) }) + return try encodeJSON(result) + case .changes(let token): + try requireAccess() + return try encodeJSON(try Library().changes(since: token)) + } + } + + /// `.readWrite` is the lowest PhotoKit level that can read assets: `.addOnly` cannot + /// enumerate or export anything. The helper never writes to the library. + static func authorize(prompt: Bool) async -> AuthorizationState { + var status = PHPhotoLibrary.authorizationStatus(for: .readWrite) + if prompt && status == .notDetermined { + status = await PHPhotoLibrary.requestAuthorization(for: .readWrite) + } + return state(status) + } + + static func state(_ status: PHAuthorizationStatus) -> AuthorizationState { + switch status { + case .authorized: return .authorized + case .limited: return .limited + case .denied: return .denied + case .restricted: return .restricted + case .notDetermined: return .notDetermined + @unknown default: return .denied + } + } + + /// Serializes progress lines written from PhotoKit callback queues and the heartbeat. + final class ProgressReporter: @unchecked Sendable { + private let lock = NSLock() + private var last = 0.0 + + /// nil repeats the last reported value. + func emit(_ fraction: Double?) { + lock.lock() + defer { lock.unlock() } + if let fraction { last = max(last, fraction) } + FileHandle.standardError.write(Data((progressLine(last) + "\n").utf8)) + } + } + + /// Every command except `authorize` runs only with granted access and never prompts. + static func requireAccess() throws { + switch state(PHPhotoLibrary.authorizationStatus(for: .readWrite)) { + case .authorized, .limited: return + default: throw HelperError(.notAuthorized, "Photos access has not been granted.") + } + } +} diff --git a/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/ArgumentsTests.swift b/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/ArgumentsTests.swift new file mode 100644 index 0000000..3bbfba3 --- /dev/null +++ b/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/ArgumentsTests.swift @@ -0,0 +1,73 @@ +import XCTest + +@testable import PhotosHelperCore + +final class ArgumentsTests: XCTestCase { + func testAuthorizePromptsUnlessAskedNotTo() throws { + XCTAssertEqual(try parseArguments(["authorize"]), .authorize(prompt: true)) + XCTAssertEqual(try parseArguments(["authorize", "--no-prompt"]), .authorize(prompt: false)) + } + + func testListDefaultsAndOptions() throws { + XCTAssertEqual(try parseArguments(["list"]), .list(cursor: nil, limit: 500, since: nil)) + let cursor = ListCursor(key: SortKey(creationMillis: 1_700_000_000_000, localIdentifier: "A/L0/001")) + let parsed = try parseArguments([ + "list", "--cursor", cursor.encoded(), "--limit", "25", "--since", "2024-01-02T03:04:05Z", + ]) + XCTAssertEqual( + parsed, + .list(cursor: cursor, limit: 25, since: Date(timeIntervalSince1970: 1_704_164_645))) + } + + func testListRejectsBadValues() { + for arguments in [ + ["list", "--limit", "0"], + ["list", "--limit", "1001"], + ["list", "--limit", "ten"], + ["list", "--limit"], + ["list", "--since", "yesterday"], + ["list", "--cursor", "not a cursor"], + ["list", "--limit", "5", "--limit", "6"], + ["list", "--dest", "/tmp"], + ["list", "extra"], + ["list", "--unknown"], + ] { + XCTAssertThrowsError(try parseArguments(arguments), "\(arguments)") { error in + XCTAssertEqual((error as? HelperError)?.code, .usage) + } + } + } + + func testAssetNeedsExactlyOneIdentifier() throws { + XCTAssertEqual(try parseArguments(["asset", "ABC/L0/001"]), .asset(localIdentifier: "ABC/L0/001")) + XCTAssertThrowsError(try parseArguments(["asset"])) + XCTAssertThrowsError(try parseArguments(["asset", "a", "b"])) + XCTAssertThrowsError(try parseArguments(["asset", String(repeating: "x", count: 201)])) + XCTAssertThrowsError(try parseArguments(["asset", "bad\nid"])) + } + + func testExportVariants() throws { + XCTAssertEqual( + try parseArguments(["export", "ID", "--dest", "/tmp/out"]), + .export(localIdentifier: "ID", destination: "/tmp/out", variant: .original)) + XCTAssertEqual( + try parseArguments(["export", "--dest", "/tmp/out", "ID", "--proxy"]), + .export(localIdentifier: "ID", destination: "/tmp/out", variant: .proxy)) + XCTAssertThrowsError(try parseArguments(["export", "ID"])) + XCTAssertThrowsError(try parseArguments(["export", "ID", "--dest", "/tmp", "--original", "--proxy"])) + } + + func testChangesNeedsToken() throws { + XCTAssertEqual(try parseArguments(["changes", "--token", "abc"]), .changes(token: "abc")) + XCTAssertThrowsError(try parseArguments(["changes"])) + XCTAssertThrowsError(try parseArguments(["changes", "--token", ""])) + } + + func testUnknownCommandIsUsageError() { + XCTAssertThrowsError(try parseArguments([])) { error in + XCTAssertEqual((error as? HelperError)?.code, .usage) + XCTAssertEqual((error as? HelperError)?.exitCode, 64) + } + XCTAssertThrowsError(try parseArguments(["delete", "ID"])) + } +} diff --git a/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/OutputTests.swift b/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/OutputTests.swift new file mode 100644 index 0000000..aa393ee --- /dev/null +++ b/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/OutputTests.swift @@ -0,0 +1,92 @@ +import XCTest + +@testable import PhotosHelperCore + +final class OutputTests: XCTestCase { + private func object(_ json: String) throws -> [String: Any] { + try XCTUnwrap(JSONSerialization.jsonObject(with: Data(json.utf8)) as? [String: Any]) + } + + func testItemEncodesEveryContractKeyWithExplicitNulls() throws { + let item = ClientMediaItem( + externalId: "ID/L0/001", kind: .photo, capturedAt: nil, durationSeconds: nil, width: 0, + height: nil, latitude: nil, longitude: nil, albumNames: [], isFavorite: false, fileName: nil) + let json = try object(encodeJSON(item)) + XCTAssertEqual( + Set(json.keys), + [ + "externalId", "kind", "capturedAt", "durationMs", "width", "height", "fps", "gpsLat", + "gpsLon", "deviceModel", "albumNames", "isFavorite", "fileName", + ]) + for key in ["capturedAt", "durationMs", "width", "height", "fps", "gpsLat", "gpsLon", "deviceModel", "fileName"] { + XCTAssertTrue(json[key] is NSNull, key) + } + } + + func testItemNormalizesValues() throws { + let item = ClientMediaItem( + externalId: "V/L0/001", kind: .livePhoto, capturedAt: Date(timeIntervalSince1970: 1_704_164_645.5), + durationSeconds: 12.3456, width: 1920, height: 1080, latitude: 52.520_08, longitude: 13.404_95, + albumNames: ["Trip", "", String(repeating: "a", count: 300)] + Array(repeating: "x", count: 60), + isFavorite: true, fileName: "IMG_0001.HEIC") + let json = try object(encodeJSON(item)) + XCTAssertEqual(json["kind"] as? String, "live_photo") + XCTAssertEqual(json["capturedAt"] as? String, "2024-01-02T03:04:05.500Z") + XCTAssertEqual(json["durationMs"] as? Int, 12_346) + XCTAssertEqual(json["gpsLat"] as? Double, 52.52) + XCTAssertEqual(json["gpsLon"] as? Double, 13.4) + let albums = try XCTUnwrap(json["albumNames"] as? [String]) + XCTAssertEqual(albums.count, 50) + XCTAssertEqual(albums[1].count, 200) + } + + func testInvalidCoordinatesAreDropped() { + XCTAssertNil(roundedCoordinate(latitude: 91, longitude: 0.5)) + XCTAssertNil(roundedCoordinate(latitude: .nan, longitude: 1)) + XCTAssertNil(roundedCoordinate(latitude: 0, longitude: 0)) + XCTAssertNil(roundedCoordinate(latitude: 1, longitude: nil)) + } + + func testClipCountsUTF16Units() { + XCTAssertEqual(clip("abc", maxCharacters: 5), "abc") + // Each emoji is two UTF-16 units; a half character is never kept. + XCTAssertEqual(clip("😀😀😀", maxCharacters: 5), "😀😀") + } + + func testErrorAndProgressLinesAreSingleLineJSON() throws { + let line = errorLine(HelperError(.notFound, "No \"asset\"\nhere.")) + XCTAssertFalse(line.contains("\n")) + let error = try XCTUnwrap(try object(line)["error"] as? [String: String]) + XCTAssertEqual(error["code"], "not_found") + XCTAssertEqual(error["message"], "No \"asset\"\nhere.") + for (input, expected) in [(0.42424, 0.424), (7, 1), (-1, 0), (.nan, 0)] as [(Double, Double)] { + let progress = progressLine(input) + XCTAssertFalse(progress.contains("\n")) + XCTAssertEqual(try object(progress)["progress"] as? Double ?? -1, expected, accuracy: 1e-9) + } + } + + func testListAndChangesKeepNullKeys() throws { + let list = try object(encodeJSON(ListOutput(items: [], nextCursor: nil, changeToken: nil))) + XCTAssertTrue(list["nextCursor"] is NSNull) + XCTAssertTrue(list["changeToken"] is NSNull) + let changes = try object(encodeJSON(ChangesOutput.fullResync())) + XCTAssertTrue(changes["token"] is NSNull) + XCTAssertEqual(changes["fullResyncRequired"] as? Bool, true) + } + + func testExitCodesAreDistinctPerFailureClass() { + let codes = HelperError.Code.allCases.map { HelperError($0, "").exitCode } + XCTAssertFalse(codes.contains(0)) + XCTAssertEqual(HelperError(.notAuthorized, "").exitCode, 77) + } + + func testSHA256StreamsFiles() throws { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: url) } + try Data("abc".utf8).write(to: url) + let digest = try sha256Hex(of: url) + XCTAssertEqual(digest.hex, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + XCTAssertEqual(digest.sizeBytes, 3) + } +} diff --git a/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/PagingTests.swift b/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/PagingTests.swift new file mode 100644 index 0000000..8248ed2 --- /dev/null +++ b/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/PagingTests.swift @@ -0,0 +1,102 @@ +import XCTest + +@testable import PhotosHelperCore + +final class PagingTests: XCTestCase { + func testCursorRoundTripsAndIsURLSafe() throws { + for key in [ + SortKey(creationMillis: 1_700_000_000_123, localIdentifier: "9F2A/L0/001"), + SortKey(creationMillis: nil, localIdentifier: "undated+/=?"), + ] { + let encoded = ListCursor(key: key).encoded() + XCTAssertFalse(encoded.contains { "+/=".contains($0) }, encoded) + XCTAssertEqual(try ListCursor.decode(encoded).key, key) + } + } + + func testCursorRejectsGarbage() { + for value in ["", "!!!", base64URLEncode(Data("{\"v\":2,\"d\":1,\"id\":\"x\"}".utf8)), + base64URLEncode(Data("{\"v\":1,\"d\":1,\"id\":\"\"}".utf8)), base64URLEncode(Data("[]".utf8))] + { + XCTAssertThrowsError(try ListCursor.decode(value), value) + } + } + + func testSortKeyPutsUndatedFirstAndBreaksTiesByIdentifier() { + let keys = [ + SortKey(creationMillis: 2, localIdentifier: "a"), + SortKey(creationMillis: 1, localIdentifier: "b"), + SortKey(creationMillis: nil, localIdentifier: "z"), + SortKey(creationMillis: 1, localIdentifier: "a"), + SortKey(creationMillis: nil, localIdentifier: "c"), + ] + XCTAssertEqual( + keys.sorted().map { key -> String in + let date = key.creationMillis.map { millis in String(millis) } ?? "-" + return date + key.localIdentifier + }, + ["-c", "-z", "1a", "1b", "2a"]) + } + + /// Simulates PhotoKit: sorted by date, ties in arbitrary (here reversed) order. + private func photoKitOrder(_ keys: [SortKey]) -> [SortKey] { + keys.sorted { lhs, rhs in + if lhs.creationMillis == rhs.creationMillis { return lhs.localIdentifier > rhs.localIdentifier } + return lhs < rhs + } + } + + private func listAll(_ source: [SortKey], limit: Int) -> [SortKey] { + var cursor: ListCursor? + var listed: [SortKey] = [] + var pages = 0 + while true { + let page = nextPage(count: source.count, key: { source[$0] }, after: cursor, limit: limit) + XCTAssertLessThanOrEqual(page.indices.count, limit) + listed += page.indices.map { source[$0] } + pages += 1 + guard page.hasMore, let last = listed.last else { break } + // The cursor survives a round trip through its encoded form, like the app stores it. + cursor = try? ListCursor.decode(ListCursor(key: last).encoded()) + if pages > source.count + 1 { XCTFail("paging did not terminate"); break } + } + return listed + } + + func testPagesCoverEverythingOnceInStableOrder() { + var keys: [SortKey] = [] + for index in 0..<57 { + // Many equal timestamps (bursts) and a few undated assets. + let millis: Int64? = index % 11 == 0 ? nil : Int64(index / 4) + keys.append(SortKey(creationMillis: millis, localIdentifier: String(format: "id-%03d", index))) + } + let source = photoKitOrder(keys) + for limit in [1, 3, 4, 10, 500] { + XCTAssertEqual(listAll(source, limit: limit), keys.sorted(), "limit \(limit)") + } + } + + func testEmptyLibraryHasNoMorePages() { + let page = nextPage(count: 0, key: { _ in fatalError() }, after: nil, limit: 10) + XCTAssertEqual(page.indices, []) + XCTAssertFalse(page.hasMore) + } + + func testCursorAfterLastItemReturnsNothing() { + let source = [SortKey(creationMillis: 1, localIdentifier: "a")] + let page = nextPage( + count: 1, key: { source[$0] }, after: ListCursor(key: source[0]), limit: 10) + XCTAssertEqual(page.indices, []) + XCTAssertFalse(page.hasMore) + } + + func testDeletedCursorItemStillResumesAfterIt() { + let source = [ + SortKey(creationMillis: 1, localIdentifier: "a"), + SortKey(creationMillis: 3, localIdentifier: "c"), + ] + let gone = ListCursor(key: SortKey(creationMillis: 2, localIdentifier: "b")) + let page = nextPage(count: 2, key: { source[$0] }, after: gone, limit: 10) + XCTAssertEqual(page.indices, [1]) + } +} diff --git a/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/PathGuardTests.swift b/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/PathGuardTests.swift new file mode 100644 index 0000000..6f431c8 --- /dev/null +++ b/apps/desktop/native/photos-helper/Tests/PhotosHelperCoreTests/PathGuardTests.swift @@ -0,0 +1,83 @@ +import XCTest + +@testable import PhotosHelperCore + +final class PathGuardTests: XCTestCase { + private var directory: URL! + + override func setUpWithError() throws { + directory = FileManager.default.temporaryDirectory.appendingPathComponent( + "photos-helper-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: directory) + } + + private func resolved() -> URL { directory.resolvingSymlinksInPath().standardizedFileURL } + + func testWritesDirectChildrenOfTheResolvedDirectory() throws { + let target = try resolveExportDestination( + directory: directory.path, fileName: "IMG_0001.MOV", temporarySuffix: "t") + XCTAssertEqual(target.finalURL.deletingLastPathComponent().path, resolved().path) + XCTAssertEqual(target.finalURL.lastPathComponent, "IMG_0001.MOV") + XCTAssertEqual(target.temporaryURL.lastPathComponent, ".IMG_0001.MOV.t.partial") + } + + func testRejectsRelativeAndTraversingDirectories() { + for path in ["relative/dir", "\(directory.path)/../x", "\(directory.path)/./", "/a/../b", ""] { + XCTAssertThrowsError( + try resolveExportDestination(directory: path, fileName: "a.jpg"), path + ) { error in + XCTAssertEqual((error as? HelperError)?.code, .pathRejected) + } + } + } + + func testRejectsMissingDirectoryAndFiles() throws { + XCTAssertThrowsError( + try resolveExportDestination(directory: directory.path + "/missing", fileName: "a.jpg")) + let file = directory.appendingPathComponent("file") + try Data().write(to: file) + XCTAssertThrowsError(try resolveExportDestination(directory: file.path, fileName: "a.jpg")) + } + + func testHostileFileNamesStayInside() throws { + for name in ["../../etc/passwd", "/etc/passwd", "..", ".hidden", "a/b\\c:d", "\u{0}x", " "] { + let target = try resolveExportDestination(directory: directory.path, fileName: name) + XCTAssertEqual(target.finalURL.deletingLastPathComponent().path, resolved().path, name) + XCTAssertFalse(target.fileName.hasPrefix("."), name) + XCTAssertFalse(target.fileName.contains("/"), name) + } + XCTAssertEqual(sanitizeFileName("../../etc/passwd"), "_.._etc_passwd") + XCTAssertEqual(sanitizeFileName(""), "asset") + } + + func testLongNamesKeepTheirExtension() { + let name = sanitizeFileName(String(repeating: "é", count: 300) + ".mov") + XCTAssertLessThanOrEqual(name.utf8.count, 200) + XCTAssertTrue(name.hasSuffix(".mov")) + } + + func testNeverOverwritesAnExistingFileOrSymlink() throws { + try Data().write(to: directory.appendingPathComponent("taken.jpg")) + XCTAssertThrowsError(try resolveExportDestination(directory: directory.path, fileName: "taken.jpg")) { + error in XCTAssertEqual((error as? HelperError)?.code, .destinationExists) + } + try FileManager.default.createSymbolicLink( + atPath: directory.appendingPathComponent("link.jpg").path, withDestinationPath: "/nonexistent") + XCTAssertThrowsError(try resolveExportDestination(directory: directory.path, fileName: "link.jpg")) + } + + func testSymlinkedDirectoryResolvesToItsTarget() throws { + let link = directory.appendingPathComponent("link", isDirectory: true) + let real = directory.appendingPathComponent("real", isDirectory: true) + try FileManager.default.createDirectory(at: real, withIntermediateDirectories: false) + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: real) + let target = try resolveExportDestination(directory: link.path, fileName: "a.jpg") + XCTAssertEqual( + target.finalURL.deletingLastPathComponent().path, + real.resolvingSymlinksInPath().standardizedFileURL.path) + } +} diff --git a/apps/desktop/native/photos-helper/entitlements.mac.plist b/apps/desktop/native/photos-helper/entitlements.mac.plist new file mode 100644 index 0000000..4699854 --- /dev/null +++ b/apps/desktop/native/photos-helper/entitlements.mac.plist @@ -0,0 +1,16 @@ + + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-library-validation + + + com.apple.security.personal-information.photos-library + + + diff --git a/apps/desktop/src/media-ipc.test.ts b/apps/desktop/src/media-ipc.test.ts new file mode 100644 index 0000000..f1d0bf7 --- /dev/null +++ b/apps/desktop/src/media-ipc.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it, vi } from "vitest"; +import type { MediaIpcEvent, MediaIpcOptions } from "./media-ipc.js"; +import { + MEDIA_PHOTOS_CHANNELS, + outcomeForError, + parseSyncRequest, + registerMediaIpc, +} from "./media-ipc.js"; +import type { PhotosHelper } from "./media-photos-helper.js"; +import { PhotosHelperError } from "./media-photos-helper.js"; +import type { PhotosSyncResult } from "./media-photos-sync.js"; +import { PhotosSyncError } from "./media-photos-sync.js"; + +type Listener = (event: MediaIpcEvent, ...args: unknown[]) => unknown; + +const result: PhotosSyncResult = { + indexed: 2, + created: 2, + updated: 0, + unchanged: 0, + uploaded: 0, + fullListing: true, +}; + +function setup(overrides: Partial = {}) { + const handlers = new Map(); + const helper = { + authorize: vi.fn(async () => "authorized" as const), + } as unknown as PhotosHelper; + const options: MediaIpcOptions = { + ipcMain: { handle: (channel, listener) => handlers.set(channel, listener) }, + platform: "darwin", + helper, + isTrusted: () => true, + runSync: vi.fn(async () => result), + ...overrides, + }; + const registration = registerMediaIpc(options); + const sent: [string, unknown][] = []; + const event: MediaIpcEvent = { + sender: { send: (channel, payload) => sent.push([channel, payload]), isDestroyed: () => false }, + }; + const call = (channel: string, ...args: unknown[]) => handlers.get(channel)!(event, ...args); + return { handlers, helper, options, registration, call, sent }; +} + +describe("media IPC on platforms without Apple Photos", () => { + it.each(["linux", "win32"])( + "reports %s as unavailable and rejects sync and cancel", + async (platform) => { + const { call, options, handlers } = setup({ platform }); + expect([...handlers.keys()].sort()).toEqual( + [ + MEDIA_PHOTOS_CHANNELS.cancel, + MEDIA_PHOTOS_CHANNELS.status, + MEDIA_PHOTOS_CHANNELS.sync, + ].sort(), + ); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toEqual({ available: false }); + expect(() => call(MEDIA_PHOTOS_CHANNELS.sync, { spaceId: null })).toThrow(/macOS/); + expect(() => call(MEDIA_PHOTOS_CHANNELS.cancel)).toThrow(/macOS/); + expect(options.runSync).not.toHaveBeenCalled(); + expect(options.helper!.authorize).not.toHaveBeenCalled(); + }, + ); + + it("treats a macOS build without the helper as unavailable", async () => { + const { call } = setup({ helper: null }); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toEqual({ available: false }); + expect(() => call(MEDIA_PHOTOS_CHANNELS.sync, {})).toThrow(/macOS/); + }); + + it("treats a helper that cannot start as unavailable", async () => { + const { call, helper } = setup(); + vi.mocked(helper.authorize).mockRejectedValueOnce(new PhotosHelperError("unavailable", "x")); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toEqual({ available: false }); + }); +}); + +describe("media IPC on macOS", () => { + it("reads status without prompting", async () => { + const { call, helper } = setup(); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toEqual({ + available: true, + authorization: "authorized", + syncing: false, + }); + expect(helper.authorize).toHaveBeenCalledWith(false); + }); + + it("runs one sync at a time, forwards progress and resolves with the outcome", async () => { + let finish: (value: PhotosSyncResult) => void = () => {}; + const runSync = vi.fn( + ({ onProgress }: Parameters[0]) => + new Promise((resolve) => { + onProgress({ phase: "indexing", indexed: 1, uploadsDone: 0, uploadsTotal: 0 }); + finish = resolve; + }), + ); + const { call, sent } = setup({ runSync }); + const first = call(MEDIA_PHOTOS_CHANNELS.sync, { spaceId: "space_1" }); + const second = call(MEDIA_PHOTOS_CHANNELS.sync, { spaceId: "space_1" }); + expect(second).toBe(first); + expect(runSync).toHaveBeenCalledTimes(1); + expect(runSync.mock.calls[0]?.[0].spaceId).toBe("space_1"); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toMatchObject({ syncing: true }); + expect(sent).toEqual([ + [ + MEDIA_PHOTOS_CHANNELS.progress, + { phase: "indexing", indexed: 1, uploadsDone: 0, uploadsTotal: 0 }, + ], + ]); + finish(result); + await expect(first).resolves.toEqual({ ok: true, result }); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).resolves.toMatchObject({ syncing: false }); + }); + + it("cancels the running sync", async () => { + const runSync = vi.fn( + ({ signal }: Parameters[0]) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => + reject(new PhotosSyncError("cancelled", "Sync cancelled.")), + ); + }), + ); + const { call } = setup({ runSync }); + const running = call(MEDIA_PHOTOS_CHANNELS.sync, {}); + await call(MEDIA_PHOTOS_CHANNELS.cancel); + await expect(running).resolves.toEqual({ + ok: false, + code: "cancelled", + message: "Sync cancelled.", + }); + }); + + it("rejects untrusted senders and malformed requests", async () => { + const { call } = setup({ isTrusted: () => false }); + expect(() => call(MEDIA_PHOTOS_CHANNELS.sync, {})).toThrow("Not allowed."); + await expect(call(MEDIA_PHOTOS_CHANNELS.status)).rejects.toThrow("Not allowed."); + expect(() => parseSyncRequest(null)).toThrow(); + expect(() => parseSyncRequest({ spaceId: "../x" })).toThrow(); + expect(() => parseSyncRequest({ spaceId: 4 })).toThrow(); + expect(parseSyncRequest({})).toEqual({ spaceId: null }); + }); + + it("never forwards unexpected error text to the renderer", () => { + expect(outcomeForError(new Error("/Users/example/secret path"))).toEqual({ + ok: false, + code: "failed", + message: "Apple Photos sync failed.", + }); + expect( + outcomeForError( + new PhotosHelperError("not_authorized", "Photos access has not been granted."), + ), + ).toMatchObject({ + code: "not_authorized", + }); + }); +}); diff --git a/apps/desktop/src/media-ipc.ts b/apps/desktop/src/media-ipc.ts new file mode 100644 index 0000000..bb8e109 --- /dev/null +++ b/apps/desktop/src/media-ipc.ts @@ -0,0 +1,135 @@ +import type { PhotosAuthorization, PhotosHelper } from "./media-photos-helper.js"; +import { PhotosHelperError } from "./media-photos-helper.js"; +import type { PhotosSyncProgress, PhotosSyncResult } from "./media-photos-sync.js"; +import { PhotosSyncError } from "./media-photos-sync.js"; + +/* + * IPC for the Apple Photos source. The renderer only expresses intent (sync, cancel) + * and renders state; the main process owns the helper, the session and the upload. + */ + +export const MEDIA_PHOTOS_CHANNELS = { + status: "desktop.media.photos.status", + sync: "desktop.media.photos.sync", + cancel: "desktop.media.photos.cancel", + progress: "desktop.media.photos.progress", +} as const; + +export type PhotosSourceStatus = + | { available: false } + | { available: true; authorization: PhotosAuthorization; syncing: boolean }; + +export type PhotosSyncOutcome = + | { ok: true; result: PhotosSyncResult } + | { ok: false; code: string; message: string }; + +/** Renderer-facing bridge, exposed as `rakazoDesktop.media.photos`. */ +export interface RakazoDesktopMediaPhotos { + status: () => Promise; + /** Asks for Photos access on first use, then indexes and uploads what the server needs. */ + sync: (request: { spaceId: string | null }) => Promise; + cancel: () => Promise; + onProgress: (listener: (progress: PhotosSyncProgress) => void) => () => void; +} + +export interface MediaIpcEvent { + sender: { send(channel: string, payload: unknown): void; isDestroyed(): boolean }; +} + +export interface MediaIpcMain { + handle(channel: string, listener: (event: MediaIpcEvent, ...args: unknown[]) => unknown): void; +} + +export interface MediaIpcOptions { + ipcMain: MediaIpcMain; + platform: string; + /** Null when the helper binary is missing (unbuilt dev checkout). */ + helper: PhotosHelper | null; + /** Same gate as the other app-window handlers: main window, main frame. */ + isTrusted: (event: MediaIpcEvent) => boolean; + runSync: (request: { + spaceId: string | null; + signal: AbortSignal; + onProgress: (progress: PhotosSyncProgress) => void; + }) => Promise; +} + +const UNAVAILABLE = "Apple Photos is only available in the macOS app."; +const SPACE_ID = /^[A-Za-z0-9_-]{1,200}$/; + +export function parseSyncRequest(value: unknown): { spaceId: string | null } { + if (typeof value !== "object" || value === null) throw new Error("Invalid sync request."); + const spaceId = (value as { spaceId?: unknown }).spaceId ?? null; + if (spaceId !== null && (typeof spaceId !== "string" || !SPACE_ID.test(spaceId))) { + throw new Error("Invalid sync request."); + } + return { spaceId }; +} + +/** Registers the handlers; `cancel()` stops a running sync, e.g. before quit. */ +export function registerMediaIpc(options: MediaIpcOptions): { cancel: () => void } { + const available = options.platform === "darwin" && options.helper !== null; + let running: { controller: AbortController; done: Promise } | null = null; + + const requireTrusted = (event: MediaIpcEvent) => { + if (!options.isTrusted(event)) throw new Error("Not allowed."); + }; + + options.ipcMain.handle(MEDIA_PHOTOS_CHANNELS.status, async (event) => { + requireTrusted(event); + if (!available) return { available: false } satisfies PhotosSourceStatus; + try { + const authorization = await options.helper!.authorize(false); + return { available: true, authorization, syncing: running !== null }; + } catch (error) { + if (error instanceof PhotosHelperError && error.code === "unavailable") { + return { available: false } satisfies PhotosSourceStatus; + } + throw error; + } + }); + + options.ipcMain.handle(MEDIA_PHOTOS_CHANNELS.sync, (event, request: unknown) => { + requireTrusted(event); + if (!available) throw new Error(UNAVAILABLE); + const { spaceId } = parseSyncRequest(request); + // A second click joins the running sync instead of starting a parallel one. + if (running) return running.done; + const controller = new AbortController(); + const done = options + .runSync({ + spaceId, + signal: controller.signal, + onProgress: (progress) => { + if (!event.sender.isDestroyed()) { + event.sender.send(MEDIA_PHOTOS_CHANNELS.progress, progress); + } + }, + }) + .then( + (result): PhotosSyncOutcome => ({ ok: true, result }), + (error: unknown): PhotosSyncOutcome => outcomeForError(error), + ) + .finally(() => { + if (running?.controller === controller) running = null; + }); + running = { controller, done }; + return done; + }); + + options.ipcMain.handle(MEDIA_PHOTOS_CHANNELS.cancel, (event) => { + requireTrusted(event); + if (!available) throw new Error(UNAVAILABLE); + running?.controller.abort(); + }); + + return { cancel: () => running?.controller.abort() }; +} + +/** Only our own fixed messages reach the renderer; anything else is generic. */ +export function outcomeForError(error: unknown): PhotosSyncOutcome { + if (error instanceof PhotosSyncError || error instanceof PhotosHelperError) { + return { ok: false, code: error.code, message: error.message }; + } + return { ok: false, code: "failed", message: "Apple Photos sync failed." }; +} diff --git a/apps/desktop/src/media-photos-helper.test.ts b/apps/desktop/src/media-photos-helper.test.ts new file mode 100644 index 0000000..513538a --- /dev/null +++ b/apps/desktop/src/media-photos-helper.test.ts @@ -0,0 +1,437 @@ +import { EventEmitter } from "node:events"; +import { mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { PassThrough } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { HelperProcess, SpawnHelper } from "./media-photos-helper.js"; +import { + changesArgs, + createPhotosHelper, + exportArgs, + helperEnv, + listArgs, + PhotosHelperError, + parseAssetOutput, + parseAuthorizeOutput, + parseChangesOutput, + parseExportOutput, + parseHelperError, + parseListOutput, + parseProgressLine, + photosHelperCandidates, + resolvePhotosHelperBinary, + runPhotosHelper, +} from "./media-photos-helper.js"; + +const item = (overrides: Record = {}) => ({ + externalId: "9F2A61B0-0000-4000-8000-000000000001/L0/001", + kind: "video", + capturedAt: "2024-01-02T03:04:05.500Z", + durationMs: 12_346, + width: 1920, + height: 1080, + fps: null, + gpsLat: 52.52, + gpsLon: 13.4, + deviceModel: null, + albumNames: ["Trip"], + isFavorite: true, + fileName: "IMG_0001.MOV", + ...overrides, +}); + +const listJson = (items: unknown[], nextCursor: unknown = null, changeToken: unknown = "tok") => + JSON.stringify({ items, nextCursor, changeToken }); + +function expectCode(run: () => unknown, code: string) { + try { + run(); + } catch (error) { + expect(error).toBeInstanceOf(PhotosHelperError); + expect((error as PhotosHelperError).code).toBe(code); + return; + } + throw new Error("expected a PhotosHelperError"); +} + +describe("photos helper output parsing", () => { + it("accepts a list page that matches the shared item contract", () => { + const page = parseListOutput( + listJson([item(), item({ kind: "photo", durationMs: null })], "c1"), + 2, + ); + expect(page.nextCursor).toBe("c1"); + expect(page.changeToken).toBe("tok"); + expect(page.items).toHaveLength(2); + expect(page.items[0]).toEqual(item()); + }); + + it("accepts the last page of an empty library", () => { + expect(parseListOutput(listJson([], null, null), 500)).toEqual({ + items: [], + nextCursor: null, + changeToken: null, + }); + }); + + it.each([ + ["not JSON", "items: []"], + ["an array document", "[]"], + ["truncated JSON", '{"items":[{"externalId":'], + ["missing items", JSON.stringify({ nextCursor: null, changeToken: null })], + ["an item missing a key", listJson([{ ...item(), width: undefined }])], + ["an unknown kind", listJson([item({ kind: "audio" })])], + ["out-of-range GPS", listJson([item({ gpsLat: 91 })])], + ["a non-UTC-parsable date", listJson([item({ capturedAt: "yesterday" })])], + ["a negative duration", listJson([item({ durationMs: -1 })])], + ["an empty external id", listJson([item({ externalId: "" })])], + ["an empty cursor", listJson([item()], "")], + ["a numeric cursor", listJson([item()], 3)], + ["too many items", listJson([item(), item(), item()])], + ])("rejects %s", (_name, text) => { + expectCode(() => parseListOutput(text, 2), "invalid_output"); + }); + + it("parses authorization states and rejects unknown ones", () => { + expect(parseAuthorizeOutput('{"status":"limited"}')).toBe("limited"); + expect(parseAuthorizeOutput('{"status":"notDetermined"}')).toBe("notDetermined"); + expectCode(() => parseAuthorizeOutput('{"status":"maybe"}'), "invalid_output"); + expectCode(() => parseAuthorizeOutput(""), "invalid_output"); + }); + + it("parses asset details", () => { + const detail = parseAssetOutput( + JSON.stringify({ + item: item(), + modifiedAt: "2024-02-01T00:00:00.000Z", + hidden: false, + resources: [{ type: "video", fileName: "IMG_0001.MOV", uti: "com.apple.quicktime-movie" }], + }), + ); + expect(detail.resources[0]?.type).toBe("video"); + expectCode( + () => parseAssetOutput(JSON.stringify({ item: item(), modifiedAt: null, resources: [] })), + "invalid_output", + ); + }); + + it("parses changes and requires a token unless a full relisting is needed", () => { + expect( + parseChangesOutput( + JSON.stringify({ + token: "t2", + changed: [item()], + deleted: ["x/L0/001"], + fullResyncRequired: false, + }), + ), + ).toMatchObject({ token: "t2", deleted: ["x/L0/001"], fullResyncRequired: false }); + expect( + parseChangesOutput( + JSON.stringify({ token: null, changed: [], deleted: [], fullResyncRequired: true }), + ).fullResyncRequired, + ).toBe(true); + expectCode( + () => + parseChangesOutput( + JSON.stringify({ token: null, changed: [], deleted: [], fullResyncRequired: false }), + ), + "invalid_output", + ); + expectCode( + () => + parseChangesOutput( + JSON.stringify({ token: "t", changed: [], deleted: [7], fullResyncRequired: false }), + ), + "invalid_output", + ); + }); + + it("maps helper error lines to codes", () => { + const stderr = + '{"progress":0.5}\n{"error":{"code":"not_authorized","message":"Photos access has not been granted."}}\n'; + const error = parseHelperError(stderr, 77); + expect(error.code).toBe("not_authorized"); + expect(error.message).toBe("Photos access has not been granted."); + expect(parseHelperError('{"error":{"code":"internal","message":"x"}}', 70).code).toBe("failed"); + expect(parseHelperError("Segmentation fault", 139).message).toContain("139"); + expect(parseHelperError('{"error":"oops"}', 1).code).toBe("failed"); + }); + + it("reads progress lines only", () => { + expect(parseProgressLine('{"progress":0.42}')).toBe(0.42); + expect(parseProgressLine('{"progress":2}')).toBeNull(); + expect(parseProgressLine('{"error":{"code":"usage"}}')).toBeNull(); + expect(parseProgressLine('{"progress":')).toBeNull(); + }); +}); + +describe("photos helper export result", () => { + let dir: string; + let other: string; + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "rakazo-photos-export-")); + other = await mkdtemp(path.join(tmpdir(), "rakazo-photos-other-")); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + await rm(other, { recursive: true, force: true }); + }); + + const result = (file: string, fileName = path.basename(file)) => + JSON.stringify({ path: file, fileName, sizeBytes: 3, sha256: "a".repeat(64) }); + + it("accepts a file directly inside the export folder", async () => { + const file = path.join(dir, "IMG_0001.MOV"); + await writeFile(file, "abc"); + await expect(parseExportOutput(result(file), dir)).resolves.toMatchObject({ + fileName: "IMG_0001.MOV", + sizeBytes: 3, + }); + }); + + it("rejects files outside the export folder, including through symlinks", async () => { + const outside = path.join(other, "secret.txt"); + await writeFile(outside, "abc"); + await expect(parseExportOutput(result(outside), dir)).rejects.toMatchObject({ + code: "path_rejected", + }); + const link = path.join(dir, "link.MOV"); + await symlink(outside, link); + await expect(parseExportOutput(result(link), dir)).rejects.toMatchObject({ + code: "path_rejected", + }); + const nested = path.join(dir, "..", path.basename(other), "secret.txt"); + await expect(parseExportOutput(result(nested), dir)).rejects.toMatchObject({ + code: "path_rejected", + }); + }); + + it("rejects malformed results", async () => { + const file = path.join(dir, "a.jpg"); + await writeFile(file, "abc"); + for (const text of [ + JSON.stringify({ path: file, fileName: "a.jpg", sizeBytes: 3, sha256: "XYZ" }), + JSON.stringify({ path: "a.jpg", fileName: "a.jpg", sizeBytes: 3, sha256: "a".repeat(64) }), + JSON.stringify({ path: file, fileName: "a.jpg", sizeBytes: -1, sha256: "a".repeat(64) }), + JSON.stringify({ + path: path.join(dir, "missing.jpg"), + fileName: "missing.jpg", + sizeBytes: 3, + sha256: "a".repeat(64), + }), + ]) { + await expect(parseExportOutput(text, dir)).rejects.toMatchObject({ code: "invalid_output" }); + } + await expect(parseExportOutput(result(file, "b.jpg"), dir)).rejects.toMatchObject({ + code: "path_rejected", + }); + }); +}); + +describe("photos helper argv", () => { + it("builds fixed argv arrays", () => { + expect(listArgs({ cursor: null, limit: 500 })).toEqual(["list", "--limit", "500"]); + expect(listArgs({ cursor: "abc_-", limit: 10, since: "2024-01-01T00:00:00Z" })).toEqual([ + "list", + "--limit", + "10", + "--cursor", + "abc_-", + "--since", + "2024-01-01T00:00:00Z", + ]); + expect(exportArgs("ID/L0/001", "/tmp/rakazo-photos-1")).toEqual([ + "export", + "ID/L0/001", + "--dest", + "/tmp/rakazo-photos-1", + "--original", + ]); + expect(changesArgs("tok")).toEqual(["changes", "--token", "tok"]); + }); + + it("keeps shell syntax as inert data and rejects option-like values", () => { + // No shell runs, so metacharacters stay one literal argument. + expect(exportArgs("$(rm -rf ~);`x`", "/tmp/a b")[1]).toBe("$(rm -rf ~);`x`"); + expectCode(() => exportArgs("--dest", "/tmp"), "usage"); + expectCode(() => listArgs({ cursor: "--limit", limit: 1 }), "usage"); + expectCode(() => changesArgs("a\nb"), "usage"); + expectCode(() => listArgs({ cursor: null, limit: 501 }), "usage"); + expectCode(() => exportArgs("ID", "relative/dir"), "path_rejected"); + expectCode(() => exportArgs("ID", "/tmp/../etc"), "path_rejected"); + }); + + it("passes only the environment PhotoKit needs", () => { + expect( + helperEnv({ HOME: "/Users/example", TMPDIR: "/tmp", OPENAI_API_KEY: "secret", PATH: "/bin" }), + ).toEqual({ HOME: "/Users/example", TMPDIR: "/tmp" }); + }); +}); + +describe("photos helper binary", () => { + it("is only available on macOS", () => { + const options = { + packaged: true, + resourcesPath: "/App/Resources", + appPath: "/app", + exists: () => true, + }; + expect(resolvePhotosHelperBinary({ ...options, platform: "linux" })).toBeNull(); + expect(resolvePhotosHelperBinary({ ...options, platform: "win32" })).toBeNull(); + expect(resolvePhotosHelperBinary({ ...options, platform: "darwin" })).toBe( + "/App/Resources/bin/photos-helper", + ); + }); + + it("falls back to the development build and reports a missing build", () => { + const options = { + platform: "darwin", + packaged: false, + resourcesPath: "/r", + appPath: "/repo/apps/desktop", + }; + expect(photosHelperCandidates(options)).toEqual([ + "/repo/apps/desktop/native/photos-helper/.build/apple/Products/Release/photos-helper", + "/repo/apps/desktop/native/photos-helper/.build/release/photos-helper", + ]); + expect( + resolvePhotosHelperBinary({ + ...options, + exists: (file) => file.endsWith(".build/release/photos-helper"), + }), + ).toBe("/repo/apps/desktop/native/photos-helper/.build/release/photos-helper"); + expect(resolvePhotosHelperBinary({ ...options, exists: () => false })).toBeNull(); + }); +}); + +class FakeProcess extends EventEmitter implements HelperProcess { + stdout = new PassThrough(); + stderr = new PassThrough(); + kill = vi.fn(() => { + queueMicrotask(() => this.emit("close", null)); + return true; + }); + exit(code: number, stdout = "", stderr = "") { + if (stdout) this.stdout.write(stdout); + if (stderr) this.stderr.write(stderr); + setImmediate(() => this.emit("close", code)); + } +} + +function fakeSpawn() { + const processes: FakeProcess[] = []; + const spawn = vi.fn(() => { + const child = new FakeProcess(); + processes.push(child); + return child; + }); + return { spawn, processes }; +} + +describe("photos helper process", () => { + it("spawns the binary with argv and no shell", async () => { + const { spawn, processes } = fakeSpawn(); + const helper = createPhotosHelper("/App/Resources/bin/photos-helper", spawn); + const pending = helper.list({ cursor: null, limit: 2 }); + processes[0]!.exit(0, listJson([item()])); + await expect(pending).resolves.toMatchObject({ items: [item()] }); + expect(spawn).toHaveBeenCalledWith( + "/App/Resources/bin/photos-helper", + ["list", "--limit", "2"], + expect.objectContaining({ shell: false, stdio: ["ignore", "pipe", "pipe"] }), + ); + }); + + it("maps a failing exit to the helper's error code", async () => { + const { spawn, processes } = fakeSpawn(); + const pending = createPhotosHelper("/bin/photos-helper", spawn).asset("ID/L0/001"); + processes[0]!.exit( + 66, + "", + '{"error":{"code":"not_found","message":"No photo or video with that identifier."}}\n', + ); + await expect(pending).rejects.toMatchObject({ code: "not_found" }); + }); + + it("reports export progress and validates the result", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "rakazo-photos-run-")); + try { + const file = path.join(dir, "IMG_0001.MOV"); + await writeFile(file, "abc"); + const { spawn, processes } = fakeSpawn(); + const progress: number[] = []; + const pending = createPhotosHelper("/bin/photos-helper", spawn).exportOriginal("ID", dir, { + onProgress: (value) => progress.push(value), + }); + const child = processes[0]!; + child.stderr.write('{"progress":0}\n{"progress":0.5'); + child.stderr.write('}\nnoise\n{"progress":1}\n'); + child.exit( + 0, + JSON.stringify({ + path: file, + fileName: "IMG_0001.MOV", + sizeBytes: 3, + sha256: "b".repeat(64), + }), + ); + await expect(pending).resolves.toMatchObject({ sizeBytes: 3 }); + expect(progress).toEqual([0, 0.5, 1]); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it("kills the helper on abort and on timeout", async () => { + const { spawn, processes } = fakeSpawn(); + const controller = new AbortController(); + const aborted = runPhotosHelper("/bin/h", ["list"], { + spawn, + timeoutMs: 60_000, + signal: controller.signal, + }); + controller.abort(); + await expect(aborted).rejects.toMatchObject({ code: "cancelled" }); + expect(processes[0]!.kill).toHaveBeenCalledWith("SIGTERM"); + + const timedOut = runPhotosHelper("/bin/h", ["list"], { spawn, timeoutMs: 5 }); + await expect(timedOut).rejects.toMatchObject({ code: "timeout" }); + expect(processes[1]!.kill).toHaveBeenCalled(); + }); + + it("fails a silent export through the idle limit", async () => { + const { spawn, processes } = fakeSpawn(); + const pending = runPhotosHelper("/bin/h", ["export"], { + spawn, + timeoutMs: 60_000, + idleTimeoutMs: 5, + }); + await expect(pending).rejects.toMatchObject({ code: "timeout" }); + expect(processes[0]!.kill).toHaveBeenCalled(); + }); + + it("does not start when already aborted and reports spawn failures", async () => { + const { spawn, processes } = fakeSpawn(); + const controller = new AbortController(); + controller.abort(); + await expect( + runPhotosHelper("/bin/h", ["list"], { spawn, timeoutMs: 1000, signal: controller.signal }), + ).rejects.toMatchObject({ code: "cancelled" }); + expect(spawn).not.toHaveBeenCalled(); + + const pending = runPhotosHelper("/bin/h", ["list"], { spawn, timeoutMs: 1000 }); + processes[0]!.emit("error", new Error("ENOENT")); + await expect(pending).rejects.toMatchObject({ code: "unavailable" }); + }); + + it("rejects oversized output", async () => { + const { spawn, processes } = fakeSpawn(); + const pending = runPhotosHelper("/bin/h", ["list"], { spawn, timeoutMs: 1000 }); + processes[0]!.stdout.write(Buffer.alloc(33 * 1024 * 1024)); + await expect(pending).rejects.toMatchObject({ code: "invalid_output" }); + }); +}); diff --git a/apps/desktop/src/media-photos-helper.ts b/apps/desktop/src/media-photos-helper.ts new file mode 100644 index 0000000..f10e402 --- /dev/null +++ b/apps/desktop/src/media-photos-helper.ts @@ -0,0 +1,517 @@ +import { spawn as nodeSpawn } from "node:child_process"; +import { realpath } from "node:fs/promises"; +import path from "node:path"; +import type { Readable } from "node:stream"; +import type { ClientMediaItem } from "@rakazo/contracts"; +import { ClientMediaItemSchema } from "@rakazo/contracts"; + +/* + * Client for the macOS `photos-helper` CLI (apps/desktop/native/photos-helper). The + * helper runs with a fixed argv and no shell; every byte it prints is validated before + * the app uses it, so a malformed or unexpected answer fails instead of being guessed at. + */ + +export type PhotosAuthorization = + | "authorized" + | "limited" + | "denied" + | "restricted" + | "notDetermined"; + +export interface PhotosListPage { + items: ClientMediaItem[]; + nextCursor: string | null; + /** Library change token from before this page was read. */ + changeToken: string | null; +} + +export interface PhotosAssetDetail { + item: ClientMediaItem; + modifiedAt: string | null; + hidden: boolean; + resources: { type: string; fileName: string; uti: string }[]; +} + +export interface PhotosChanges { + token: string | null; + changed: ClientMediaItem[]; + deleted: string[]; + fullResyncRequired: boolean; +} + +export interface PhotosExport { + path: string; + fileName: string; + sizeBytes: number; + sha256: string; +} + +export type PhotosHelperErrorCode = + | "unavailable" + | "usage" + | "not_authorized" + | "not_found" + | "path_rejected" + | "destination_exists" + | "network" + | "export_failed" + | "invalid_output" + | "timeout" + | "cancelled" + | "failed"; + +const HELPER_CODES = new Set([ + "usage", + "not_authorized", + "not_found", + "path_rejected", + "destination_exists", + "network", + "export_failed", +]); + +export class PhotosHelperError extends Error { + constructor( + readonly code: PhotosHelperErrorCode, + message: string, + ) { + super(message); + this.name = "PhotosHelperError"; + } +} + +export const PHOTOS_HELPER_NAME = "photos-helper"; +/** Matches the contract's batch limit for `media.sources.pushItems`. */ +export const PHOTOS_MAX_PAGE = 500; + +/** + * Packaged apps ship the universal helper in Resources/bin; development uses the + * SwiftPM output (universal `--arch` build first, then the host-arch build). + */ +export function photosHelperCandidates(options: { + packaged: boolean; + resourcesPath: string; + appPath: string; +}): string[] { + if (options.packaged) return [path.join(options.resourcesPath, "bin", PHOTOS_HELPER_NAME)]; + const build = path.join(options.appPath, "native", "photos-helper", ".build"); + return [ + path.join(build, "apple", "Products", "Release", PHOTOS_HELPER_NAME), + path.join(build, "release", PHOTOS_HELPER_NAME), + ]; +} + +/** Null off macOS or when the helper was not built; the Apple Photos source is then hidden. */ +export function resolvePhotosHelperBinary(options: { + platform: string; + packaged: boolean; + resourcesPath: string; + appPath: string; + exists: (file: string) => boolean; +}): string | null { + if (options.platform !== "darwin") return null; + return photosHelperCandidates(options).find((candidate) => options.exists(candidate)) ?? null; +} + +// --------------------------------------------------------------------------- +// argv +// --------------------------------------------------------------------------- + +export function authorizeArgs(prompt: boolean): string[] { + return prompt ? ["authorize"] : ["authorize", "--no-prompt"]; +} + +export function listArgs(request: { + cursor: string | null; + limit: number; + since?: string | null; +}): string[] { + if (!Number.isInteger(request.limit) || request.limit < 1 || request.limit > PHOTOS_MAX_PAGE) { + throw new PhotosHelperError("usage", "List limit is out of range."); + } + const args = ["list", "--limit", String(request.limit)]; + if (request.cursor !== null) args.push("--cursor", opaqueArgument(request.cursor)); + if (request.since) args.push("--since", opaqueArgument(request.since)); + return args; +} + +export function assetArgs(localIdentifier: string): string[] { + return ["asset", identifierArgument(localIdentifier)]; +} + +export function exportArgs(localIdentifier: string, destination: string): string[] { + if (!path.isAbsolute(destination) || destination.split(/[\\/]/).includes("..")) { + throw new PhotosHelperError("path_rejected", "Export directory must be absolute."); + } + return ["export", identifierArgument(localIdentifier), "--dest", destination, "--original"]; +} + +export function changesArgs(token: string): string[] { + return ["changes", "--token", opaqueArgument(token)]; +} + +/** Identifiers come from the helper itself; a leading dash would read as an option. */ +function identifierArgument(value: string): string { + if (value === "" || value.length > 200 || value.startsWith("-") || hasControl(value)) { + throw new PhotosHelperError("usage", "Invalid Photos identifier."); + } + return value; +} + +function opaqueArgument(value: string): string { + if (value === "" || value.length > 4096 || value.startsWith("-") || hasControl(value)) { + throw new PhotosHelperError("usage", "Invalid helper argument."); + } + return value; +} + +function hasControl(value: string): boolean { + for (const char of value) { + const code = char.codePointAt(0) ?? 0; + if (code < 0x20 || code === 0x7f) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// Output parsing +// --------------------------------------------------------------------------- + +function invalid(what: string): never { + throw new PhotosHelperError("invalid_output", `Photos helper returned invalid ${what}.`); +} + +function parseDocument(text: string): Record { + let value: unknown; + try { + value = JSON.parse(text); + } catch { + invalid("JSON"); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) invalid("JSON"); + return value as Record; +} + +function nullableString(value: unknown, what: string, max = 4096): string | null { + if (value === null) return null; + if (typeof value !== "string" || value === "" || value.length > max) invalid(what); + return value; +} + +/** Each item must satisfy the shared contract schema exactly as the server will parse it. */ +function parseItems(value: unknown, what: string): ClientMediaItem[] { + if (!Array.isArray(value) || value.length > 10_000) invalid(what); + return value.map((entry) => { + if (typeof entry !== "object" || entry === null) invalid(what); + const parsed = ClientMediaItemSchema.safeParse(entry); + if (!parsed.success) invalid(what); + return parsed.data; + }); +} + +const AUTHORIZATION = new Set([ + "authorized", + "limited", + "denied", + "restricted", + "notDetermined", +]); + +export function parseAuthorizeOutput(text: string): PhotosAuthorization { + const status = parseDocument(text).status; + if (typeof status !== "string" || !AUTHORIZATION.has(status as PhotosAuthorization)) { + invalid("authorization status"); + } + return status as PhotosAuthorization; +} + +export function parseListOutput(text: string, limit: number): PhotosListPage { + const document = parseDocument(text); + const items = parseItems(document.items, "list items"); + if (items.length > limit) invalid("list page size"); + const nextCursor = nullableString(document.nextCursor, "list cursor"); + const changeToken = nullableString(document.changeToken, "change token"); + return { items, nextCursor, changeToken }; +} + +export function parseAssetOutput(text: string): PhotosAssetDetail { + const document = parseDocument(text); + const [item] = parseItems([document.item], "asset"); + const modifiedAt = nullableString(document.modifiedAt, "asset date", 64); + if (modifiedAt !== null && Number.isNaN(Date.parse(modifiedAt))) invalid("asset date"); + if (typeof document.hidden !== "boolean") invalid("asset"); + if (!Array.isArray(document.resources) || document.resources.length > 50) invalid("resources"); + const resources = document.resources.map((resource) => { + if (typeof resource !== "object" || resource === null) invalid("resources"); + const { type, fileName, uti } = resource as Record; + if (typeof type !== "string" || typeof fileName !== "string" || typeof uti !== "string") { + invalid("resources"); + } + return { type, fileName, uti }; + }); + return { item: item!, modifiedAt, hidden: document.hidden, resources }; +} + +export function parseChangesOutput(text: string): PhotosChanges { + const document = parseDocument(text); + const token = nullableString(document.token, "change token"); + if (typeof document.fullResyncRequired !== "boolean") invalid("changes"); + const changed = parseItems(document.changed, "changed items"); + if (!Array.isArray(document.deleted) || document.deleted.length > 10_000) invalid("changes"); + const deleted = document.deleted.map((id) => { + if (typeof id !== "string" || id === "" || id.length > 200) invalid("deleted ids"); + return id; + }); + if (!document.fullResyncRequired && token === null) invalid("change token"); + return { token, changed, deleted, fullResyncRequired: document.fullResyncRequired }; +} + +/** + * Besides the shape, the file must be a direct child of the directory the app chose, + * after resolving symlinks, so a confused helper cannot point the upload elsewhere. + */ +export async function parseExportOutput(text: string, destination: string): Promise { + const document = parseDocument(text); + const { path: file, fileName, sizeBytes, sha256 } = document; + if ( + typeof file !== "string" || + !path.isAbsolute(file) || + typeof fileName !== "string" || + fileName === "" || + typeof sizeBytes !== "number" || + !Number.isSafeInteger(sizeBytes) || + sizeBytes < 0 || + typeof sha256 !== "string" || + !/^[0-9a-f]{64}$/.test(sha256) + ) { + invalid("export result"); + } + let real: string; + let realDestination: string; + try { + [real, realDestination] = await Promise.all([realpath(file), realpath(destination)]); + } catch { + invalid("export path"); + } + if (path.dirname(real) !== realDestination || path.basename(real) !== fileName) { + throw new PhotosHelperError("path_rejected", "Photos helper wrote outside the export folder."); + } + return { path: real, fileName, sizeBytes, sha256 }; +} + +/** The last `{"error":{code,message}}` line on stderr; unknown codes become `failed`. */ +export function parseHelperError(stderr: string, exitCode: number | null): PhotosHelperError { + const lines = stderr.split("\n").reverse(); + for (const line of lines) { + if (!line.startsWith("{")) continue; + try { + const error = (JSON.parse(line) as { error?: unknown }).error as + | { code?: unknown; message?: unknown } + | undefined; + if (typeof error?.code !== "string") continue; + const code = HELPER_CODES.has(error.code as PhotosHelperErrorCode) + ? (error.code as PhotosHelperErrorCode) + : "failed"; + const message = + typeof error.message === "string" && error.message.length <= 500 + ? error.message + : "Photos helper failed."; + return new PhotosHelperError(code, message); + } catch {} + } + return new PhotosHelperError("failed", `Photos helper exited with status ${exitCode ?? "?"}.`); +} + +/** `{"progress":0.42}` lines on stderr; anything else is ignored. */ +export function parseProgressLine(line: string): number | null { + if (!line.startsWith('{"progress"')) return null; + try { + const value = (JSON.parse(line) as { progress?: unknown }).progress; + return typeof value === "number" && value >= 0 && value <= 1 ? value : null; + } catch { + return null; + } +} + +// --------------------------------------------------------------------------- +// Process +// --------------------------------------------------------------------------- + +/** The subset of ChildProcess the runner needs, so tests can supply a fake. */ +export interface HelperProcess { + stdout: Readable | null; + stderr: Readable | null; + kill(signal?: NodeJS.Signals): boolean; + on(event: "error", listener: (error: Error) => void): unknown; + on(event: "close", listener: (code: number | null) => void): unknown; +} + +export type SpawnHelper = ( + binary: string, + args: string[], + options: { shell: false; env: Record; stdio: ["ignore", "pipe", "pipe"] }, +) => HelperProcess; + +export interface RunOptions { + signal?: AbortSignal; + /** Hard limit for the whole run. */ + timeoutMs: number; + /** Fails when stderr progress stops for this long (exports only). */ + idleTimeoutMs?: number; + onProgress?: (fraction: number) => void; +} + +const MAX_STDOUT_BYTES = 32 * 1024 * 1024; +const MAX_STDERR_BYTES = 1024 * 1024; +const ENV_ALLOWLIST = ["HOME", "USER", "LOGNAME", "TMPDIR", "LANG", "LC_ALL"]; + +/** PhotoKit needs the user's home; nothing else from the app environment is passed on. */ +export function helperEnv(env: NodeJS.ProcessEnv): Record { + const result: Record = {}; + for (const name of ENV_ALLOWLIST) { + const value = env[name]; + if (value) result[name] = value; + } + return result; +} + +export const defaultSpawnHelper: SpawnHelper = (binary, args, options) => + nodeSpawn(binary, args, options); + +export function runPhotosHelper( + binary: string, + args: string[], + options: RunOptions & { spawn?: SpawnHelper; env?: NodeJS.ProcessEnv }, +): Promise { + return new Promise((resolve, reject) => { + if (options.signal?.aborted) { + reject(new PhotosHelperError("cancelled", "Cancelled.")); + return; + } + const child = (options.spawn ?? defaultSpawnHelper)(binary, args, { + shell: false, + env: helperEnv(options.env ?? process.env), + stdio: ["ignore", "pipe", "pipe"], + }); + const stdout: Buffer[] = []; + let stdoutBytes = 0; + let stderr = ""; + let pendingLine = ""; + let settled = false; + let idleTimer: NodeJS.Timeout | undefined; + + const finish = (error: PhotosHelperError | null, output = "") => { + if (settled) return; + settled = true; + clearTimeout(timer); + clearTimeout(idleTimer); + options.signal?.removeEventListener("abort", abort); + if (error) reject(error); + else resolve(output); + }; + const stop = (error: PhotosHelperError) => { + child.kill("SIGTERM"); + finish(error); + }; + const abort = () => stop(new PhotosHelperError("cancelled", "Cancelled.")); + const timer = setTimeout( + () => stop(new PhotosHelperError("timeout", "Photos helper timed out.")), + options.timeoutMs, + ); + timer.unref?.(); + const armIdle = () => { + if (options.idleTimeoutMs === undefined) return; + clearTimeout(idleTimer); + idleTimer = setTimeout( + () => stop(new PhotosHelperError("timeout", "Photos export stalled.")), + options.idleTimeoutMs, + ); + idleTimer.unref?.(); + }; + armIdle(); + options.signal?.addEventListener("abort", abort, { once: true }); + + child.stdout?.on("data", (chunk: Buffer) => { + stdoutBytes += chunk.byteLength; + if (stdoutBytes > MAX_STDOUT_BYTES) { + stop(new PhotosHelperError("invalid_output", "Photos helper output is too large.")); + return; + } + stdout.push(chunk); + }); + child.stderr?.on("data", (chunk: Buffer) => { + const text = chunk.toString("utf8"); + if (stderr.length < MAX_STDERR_BYTES) stderr += text; + const lines = (pendingLine + text).split("\n"); + pendingLine = lines.pop() ?? ""; + for (const line of lines) { + const progress = parseProgressLine(line.trim()); + if (progress !== null) { + armIdle(); + options.onProgress?.(progress); + } + } + }); + child.on("error", () => { + finish(new PhotosHelperError("unavailable", "Photos helper could not be started.")); + }); + child.on("close", (code) => { + if (code === 0) finish(null, Buffer.concat(stdout).toString("utf8")); + else finish(parseHelperError(stderr + pendingLine, code)); + }); + }); +} + +const QUERY_TIMEOUT_MS = 2 * 60_000; +/** The system permission dialog waits on the person. */ +const AUTHORIZE_TIMEOUT_MS = 10 * 60_000; +/** Large iCloud videos take long; a stalled download fails sooner through the idle limit. */ +const EXPORT_TIMEOUT_MS = 2 * 60 * 60_000; +const EXPORT_IDLE_TIMEOUT_MS = 5 * 60_000; + +export interface PhotosHelper { + authorize(prompt: boolean, signal?: AbortSignal): Promise; + list( + request: { cursor: string | null; limit: number; since?: string | null }, + signal?: AbortSignal, + ): Promise; + asset(localIdentifier: string, signal?: AbortSignal): Promise; + exportOriginal( + localIdentifier: string, + destination: string, + options?: { signal?: AbortSignal; onProgress?: (fraction: number) => void }, + ): Promise; + changes(token: string, signal?: AbortSignal): Promise; +} + +export function createPhotosHelper(binary: string, spawn?: SpawnHelper): PhotosHelper { + const run = (args: string[], options: RunOptions) => + runPhotosHelper(binary, args, { ...options, spawn }); + return { + async authorize(prompt, signal) { + const timeoutMs = prompt ? AUTHORIZE_TIMEOUT_MS : QUERY_TIMEOUT_MS; + return parseAuthorizeOutput(await run(authorizeArgs(prompt), { signal, timeoutMs })); + }, + async list(request, signal) { + const output = await run(listArgs(request), { signal, timeoutMs: QUERY_TIMEOUT_MS }); + return parseListOutput(output, request.limit); + }, + async asset(localIdentifier, signal) { + const output = await run(assetArgs(localIdentifier), { signal, timeoutMs: QUERY_TIMEOUT_MS }); + return parseAssetOutput(output); + }, + async exportOriginal(localIdentifier, destination, options = {}) { + const output = await run(exportArgs(localIdentifier, destination), { + signal: options.signal, + onProgress: options.onProgress, + timeoutMs: EXPORT_TIMEOUT_MS, + idleTimeoutMs: EXPORT_IDLE_TIMEOUT_MS, + }); + return parseExportOutput(output, destination); + }, + async changes(token, signal) { + const output = await run(changesArgs(token), { signal, timeoutMs: QUERY_TIMEOUT_MS }); + return parseChangesOutput(output); + }, + }; +} diff --git a/apps/desktop/src/media-photos-sync.test.ts b/apps/desktop/src/media-photos-sync.test.ts new file mode 100644 index 0000000..d37b569 --- /dev/null +++ b/apps/desktop/src/media-photos-sync.test.ts @@ -0,0 +1,655 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { ClientMediaItem, MediaAsset, MediaUpload } from "@rakazo/contracts"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { PhotosHelper } from "./media-photos-helper.js"; +import type { MediaSyncApi, PhotosSyncState, PhotosSyncStore } from "./media-photos-sync.js"; +import { + createChunkUploader, + createPhotosSyncFileStore, + parsePhotosSyncState, + runPool, + syncApplePhotos, +} from "./media-photos-sync.js"; + +const sha = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); + +function libraryItem(index: number): ClientMediaItem { + return { + externalId: `ASSET-${String(index).padStart(5, "0")}/L0/001`, + kind: index % 3 === 0 ? "video" : "photo", + capturedAt: new Date(Date.UTC(2024, 0, 1) + index * 60_000).toISOString(), + durationMs: index % 3 === 0 ? 5000 : null, + width: 1920, + height: 1080, + fps: null, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + isFavorite: false, + fileName: `IMG_${index}.JPG`, + }; +} + +/** Pages a synthetic library the way the Swift helper does: opaque cursor = next index. */ +function fakeHelper(options: { size: number; contents?: (id: string) => string }) { + const library = Array.from({ length: options.size }, (_, index) => libraryItem(index)); + const exports: string[] = []; + let active = 0; + const stats = { maxActiveExports: 0 }; + const helper: PhotosHelper & { library: ClientMediaItem[] } = { + library, + authorize: vi.fn(async () => "authorized" as const), + list: vi.fn(async ({ cursor, limit }) => { + const start = cursor === null ? 0 : Number(cursor); + const items = library.slice(start, start + limit); + const next = start + limit < library.length ? String(start + limit) : null; + return { items, nextCursor: next, changeToken: `token-${library.length}` }; + }), + asset: vi.fn(), + changes: vi.fn(), + exportOriginal: vi.fn(async (id: string, destination: string, opts) => { + active += 1; + stats.maxActiveExports = Math.max(stats.maxActiveExports, active); + try { + await new Promise((resolve) => setTimeout(resolve, 5)); + opts?.signal?.throwIfAborted(); + const contents = options.contents?.(id) ?? `original bytes of ${id}`; + const file = path.join(destination, "IMG.MOV"); + await writeFile(file, contents); + exports.push(destination); + return { + path: file, + fileName: "IMG.MOV", + sizeBytes: Buffer.byteLength(contents), + sha256: sha(contents), + }; + } finally { + active -= 1; + } + }), + }; + return { helper, exports, stats }; +} + +function memoryStore(initial?: PhotosSyncState) { + let state: PhotosSyncState = initial ?? { version: 1, deviceId: "device-1", targets: {} }; + const writes: PhotosSyncState[] = []; + const store: PhotosSyncStore = { + read: async () => JSON.parse(JSON.stringify(state)) as PhotosSyncState, + write: async (next) => { + state = JSON.parse(JSON.stringify(next)) as PhotosSyncState; + writes.push(state); + }, + }; + return { store, writes, current: () => state }; +} + +function asset(id: string, externalId: string, ingestState: MediaAsset["ingestState"]) { + return { + id, + sourceId: "source-1", + externalId, + kind: "video", + storage: "remote", + capturedAt: null, + durationMs: null, + width: null, + height: null, + fps: null, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + isFavorite: false, + ingestState, + hasProxy: false, + hasThumbnail: false, + qualityScore: null, + createdAt: "2024-01-01T00:00:00.000Z", + } as MediaAsset; +} + +function fakeApi( + options: { + assets?: MediaAsset[]; + chunkBytes?: number; + uploads?: Record; + } = {}, +) { + const uploads = new Map(Object.entries(options.uploads ?? {})); + const pushes: { + items: number; + syncToken: string | null; + complete: boolean; + deleted?: number; + }[] = []; + const chunkBytes = options.chunkBytes ?? 4; + const api = { + media: { + sources: { + registerClient: vi.fn(async (input: { deviceId: string }) => ({ + id: "source-1", + kind: "apple_photos" as const, + label: "Apple Photos", + status: "idle" as const, + assetCount: 0, + lastSyncedAt: null, + error: null, + createdAt: "2024-01-01T00:00:00.000Z", + deviceId: input.deviceId, + })), + pushItems: vi.fn( + async (input: { + items: unknown[]; + deletedExternalIds?: string[]; + syncToken: string | null; + complete: boolean; + }) => { + const deleted = input.deletedExternalIds?.length ?? 0; + expect(input.items.length > 0 || deleted > 0 || input.complete).toBe(true); + expect(input.items.length).toBeLessThanOrEqual(500); + pushes.push({ + items: input.items.length, + syncToken: input.syncToken, + complete: input.complete, + ...(deleted > 0 ? { deleted } : {}), + }); + return { created: input.items.length, updated: 0, unchanged: 0, deleted }; + }, + ), + }, + assets: { + search: vi.fn(async (input: { cursor?: string | null }) => { + const all = options.assets ?? []; + // Two pages to exercise pagination. + const half = Math.ceil(all.length / 2); + return input.cursor === "p2" + ? { items: all.slice(half), nextCursor: null } + : { items: all.slice(0, half), nextCursor: all.length > 1 ? "p2" : null }; + }), + }, + uploads: { + begin: vi.fn(async (input: { assetId: string; sizeBytes: number }) => { + const upload: MediaUpload = { + id: `upload-${input.assetId}`, + assetId: input.assetId, + sizeBytes: input.sizeBytes, + chunkBytes, + receivedChunks: [], + status: "open", + }; + uploads.set(upload.id, upload); + return upload; + }), + status: vi.fn(async (input: { uploadId: string }) => { + const upload = uploads.get(input.uploadId); + if (!upload) throw Object.assign(new Error("Not found"), { code: "NOT_FOUND" }); + return upload; + }), + complete: vi.fn(async (input: { uploadId: string }) => { + const upload = uploads.get(input.uploadId)!; + return { ...upload, status: "complete" as const }; + }), + }, + }, + }; + return { api: api as unknown as MediaSyncApi & typeof api, pushes }; +} + +let tempRoot: string; + +beforeEach(async () => { + tempRoot = await mkdtemp(path.join(tmpdir(), "rakazo-photos-sync-")); +}); + +afterEach(async () => { + await rm(tempRoot, { recursive: true, force: true }); +}); + +const baseOptions = () => ({ + target: "https://rakazo.example.com#space_1", + label: "Apple Photos", + tempRoot, + signal: new AbortController().signal, +}); + +describe("Apple Photos metadata sync", () => { + it("registers once, pushes batches of at most 500 and completes the listing", async () => { + const { helper } = fakeHelper({ size: 1203 }); + const { api, pushes } = fakeApi(); + const { store, current } = memoryStore(); + const chunks = vi.fn(); + + const result = await syncApplePhotos({ + ...baseOptions(), + helper, + api, + store, + uploadChunk: chunks, + }); + + expect(api.media.sources.registerClient).toHaveBeenCalledWith( + { kind: "apple_photos", label: "Apple Photos", deviceId: "device-1" }, + expect.anything(), + ); + expect(pushes).toEqual([ + { items: 500, syncToken: null, complete: false }, + { items: 500, syncToken: null, complete: false }, + { items: 203, syncToken: "token-1203", complete: true }, + ]); + expect(result).toMatchObject({ indexed: 1203, created: 1203, uploaded: 0, fullListing: true }); + expect(current().targets["https://rakazo.example.com#space_1"]).toEqual({ + sourceId: "source-1", + listing: null, + changeToken: "token-1203", + uploads: {}, + }); + + // The next run reads changes only and does not register again. + vi.mocked(helper.changes).mockResolvedValueOnce({ + token: "token-next", + changed: [libraryItem(1), libraryItem(2)], + deleted: [], + fullResyncRequired: false, + }); + const second = await syncApplePhotos({ + ...baseOptions(), + helper, + api, + store, + uploadChunk: chunks, + }); + expect(api.media.sources.registerClient).toHaveBeenCalledTimes(1); + expect(helper.changes).toHaveBeenCalledWith("token-1203", expect.anything()); + expect(pushes.at(-1)).toEqual({ items: 2, syncToken: "token-next", complete: false }); + expect(second.fullListing).toBe(false); + expect(current().targets["https://rakazo.example.com#space_1"]?.changeToken).toBe("token-next"); + }); + + it("persists the cursor after every batch and resumes an interrupted listing", async () => { + const { helper } = fakeHelper({ size: 1100 }); + const { api, pushes } = fakeApi(); + const { store, writes, current } = memoryStore(); + vi.mocked(api.media.sources.pushItems).mockImplementationOnce(async (input) => { + pushes.push({ + items: input.items.length, + syncToken: input.syncToken, + complete: input.complete, + }); + return { created: input.items.length, updated: 0, unchanged: 0 }; + }); + vi.mocked(api.media.sources.pushItems).mockRejectedValueOnce(new Error("network down")); + + await expect( + syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }), + ).rejects.toThrow("network down"); + const interrupted = current().targets["https://rakazo.example.com#space_1"]!; + expect(interrupted.listing).toMatchObject({ cursor: "500", token: "token-1100" }); + expect(interrupted.listing?.lastItem?.externalId).toBe(libraryItem(499).externalId); + expect(writes.length).toBeGreaterThan(1); + + await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }); + expect(vi.mocked(helper.list).mock.calls.at(-2)?.[0]).toMatchObject({ cursor: "500" }); + expect(pushes.slice(1)).toEqual([ + { items: 500, syncToken: null, complete: false }, + { items: 100, syncToken: "token-1100", complete: true }, + ]); + expect(current().targets["https://rakazo.example.com#space_1"]).toMatchObject({ + listing: null, + changeToken: "token-1100", + }); + }); + + it("closes a resumed listing whose last page came back empty", async () => { + const { helper } = fakeHelper({ size: 0 }); + const { api, pushes } = fakeApi(); + const last = libraryItem(7); + const { store } = memoryStore({ + version: 1, + deviceId: "device-1", + targets: { + "https://rakazo.example.com#space_1": { + sourceId: "source-1", + listing: { cursor: "500", token: "token-a", lastItem: last }, + changeToken: null, + uploads: {}, + }, + }, + }); + await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }); + expect(pushes).toEqual([{ items: 0, syncToken: "token-a", complete: true }]); + }); + + it("pushes device deletions without relisting", async () => { + const { helper } = fakeHelper({ size: 3 }); + const { api, pushes } = fakeApi(); + const { store } = memoryStore({ + version: 1, + deviceId: "device-1", + targets: { + "https://rakazo.example.com#space_1": { + sourceId: "source-1", + listing: null, + changeToken: "old", + uploads: {}, + }, + }, + }); + vi.mocked(helper.changes).mockResolvedValueOnce({ + token: "new", + changed: [], + deleted: ["gone/L0/001"], + fullResyncRequired: false, + }); + const result = await syncApplePhotos({ + ...baseOptions(), + helper, + api, + store, + uploadChunk: vi.fn(), + }); + expect(result.fullListing).toBe(false); + expect(pushes).toEqual([{ items: 0, syncToken: "new", complete: false, deleted: 1 }]); + }); + + it("registers again when the server no longer knows the source", async () => { + const { helper } = fakeHelper({ size: 2 }); + const { api } = fakeApi(); + const { store } = memoryStore({ + version: 1, + deviceId: "device-1", + targets: { + "https://rakazo.example.com#space_1": { + sourceId: "deleted-source", + listing: { cursor: null, token: null, lastItem: null }, + changeToken: null, + uploads: {}, + }, + }, + }); + vi.mocked(api.media.sources.pushItems).mockRejectedValueOnce( + Object.assign(new Error("Not found"), { code: "NOT_FOUND" }), + ); + await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk: vi.fn() }); + expect(api.media.sources.registerClient).toHaveBeenCalledTimes(1); + expect(vi.mocked(api.media.sources.pushItems).mock.calls.at(-1)?.[0].sourceId).toBe("source-1"); + }); + + it("stops before anything is sent when Photos access is denied", async () => { + const { helper } = fakeHelper({ size: 2 }); + vi.mocked(helper.authorize).mockResolvedValueOnce("denied"); + const { api } = fakeApi(); + await expect( + syncApplePhotos({ + ...baseOptions(), + helper, + api, + store: memoryStore().store, + uploadChunk: vi.fn(), + }), + ).rejects.toMatchObject({ code: "not_authorized" }); + expect(api.media.sources.registerClient).not.toHaveBeenCalled(); + }); +}); + +describe("Apple Photos original upload", () => { + const awaitingAssets = [ + asset("a1", "ASSET-00001/L0/001", "awaiting_upload"), + asset("a2", "ASSET-00002/L0/001", "proxied"), + asset("a3", "ASSET-00003/L0/001", "awaiting_upload"), + asset("a4", "ASSET-00004/L0/001", "awaiting_upload"), + ]; + + it("uploads every awaiting original in chunks with two at a time and cleans up", async () => { + const { helper, exports, stats } = fakeHelper({ size: 1 }); + const { api } = fakeApi({ assets: awaitingAssets, chunkBytes: 8 }); + const sent = new Map(); + const uploadChunk = vi.fn(async (uploadId: string, index: number, bytes: Uint8Array) => { + const parts = sent.get(uploadId) ?? []; + parts[index] = Buffer.from(bytes); + sent.set(uploadId, parts); + }); + const progress = vi.fn(); + + const result = await syncApplePhotos({ + ...baseOptions(), + helper, + api, + store: memoryStore().store, + uploadChunk, + onProgress: progress, + }); + + expect(result.uploaded).toBe(3); + expect( + vi + .mocked(helper.exportOriginal) + .mock.calls.map((call) => call[0]) + .sort(), + ).toEqual(["ASSET-00001/L0/001", "ASSET-00003/L0/001", "ASSET-00004/L0/001"]); + expect(stats.maxActiveExports).toBe(2); + expect(Buffer.concat(sent.get("upload-a1")!).toString()).toBe( + "original bytes of ASSET-00001/L0/001", + ); + expect(sent.get("upload-a1")).toHaveLength(Math.ceil(36 / 8)); + expect(api.media.uploads.begin).toHaveBeenCalledWith( + expect.objectContaining({ + assetId: "a1", + sizeBytes: 36, + sha256: sha("original bytes of ASSET-00001/L0/001"), + }), + expect.anything(), + ); + expect(api.media.uploads.complete).toHaveBeenCalledTimes(3); + expect(progress).toHaveBeenLastCalledWith({ + phase: "uploading", + indexed: 1, + uploadsDone: 3, + uploadsTotal: 3, + }); + for (const dir of exports) await expect(readdir(dir)).rejects.toThrow(); + await expect(readdir(tempRoot)).resolves.toEqual([]); + }); + + it("resumes an open upload and sends only the missing chunks", async () => { + const { helper } = fakeHelper({ size: 1 }); + const contents = "original bytes of ASSET-00001/L0/001"; + const { api } = fakeApi({ + assets: [awaitingAssets[0]!], + uploads: { + "upload-old": { + id: "upload-old", + assetId: "a1", + sizeBytes: Buffer.byteLength(contents), + chunkBytes: 10, + receivedChunks: [0, 2], + status: "open", + }, + }, + }); + const { store, current } = memoryStore({ + version: 1, + deviceId: "device-1", + targets: { + "https://rakazo.example.com#space_1": { + sourceId: "source-1", + listing: null, + changeToken: "tok", + uploads: { a1: "upload-old" }, + }, + }, + }); + vi.mocked(helper.changes).mockResolvedValueOnce({ + token: "tok", + changed: [], + deleted: [], + fullResyncRequired: false, + }); + const uploadChunk = vi.fn(async () => {}); + + await syncApplePhotos({ ...baseOptions(), helper, api, store, uploadChunk }); + + expect(api.media.uploads.begin).not.toHaveBeenCalled(); + expect( + uploadChunk.mock.calls.map((call) => [ + call[0], + call[1], + Buffer.from(call[2] as Uint8Array).toString(), + ]), + ).toEqual([ + ["upload-old", 1, contents.slice(10, 20)], + ["upload-old", 3, contents.slice(30)], + ]); + expect(current().targets["https://rakazo.example.com#space_1"]?.uploads).toEqual({}); + }); + + it("aborts on a checksum mismatch without starting an upload", async () => { + const { helper } = fakeHelper({ size: 1 }); + vi.mocked(helper.exportOriginal).mockImplementationOnce(async (_id, destination) => { + const file = path.join(destination, "IMG.MOV"); + await writeFile(file, "tampered"); + return { path: file, fileName: "IMG.MOV", sizeBytes: 8, sha256: sha("original") }; + }); + const { api } = fakeApi({ assets: [awaitingAssets[0]!] }); + const uploadChunk = vi.fn(); + + await expect( + syncApplePhotos({ ...baseOptions(), helper, api, store: memoryStore().store, uploadChunk }), + ).rejects.toMatchObject({ code: "integrity" }); + expect(api.media.uploads.begin).not.toHaveBeenCalled(); + expect(uploadChunk).not.toHaveBeenCalled(); + await expect(readdir(tempRoot)).resolves.toEqual([]); + }); + + it("cancels a running upload", async () => { + const { helper } = fakeHelper({ size: 1 }); + const { api } = fakeApi({ assets: [awaitingAssets[0]!], chunkBytes: 4 }); + const controller = new AbortController(); + const uploadChunk = vi.fn( + async (_id: string, index: number, _bytes: Uint8Array, signal: AbortSignal) => { + if (index === 1) controller.abort(); + signal.throwIfAborted(); + }, + ); + + await expect( + syncApplePhotos({ + ...baseOptions(), + signal: controller.signal, + helper, + api, + store: memoryStore().store, + uploadChunk, + }), + ).rejects.toMatchObject({ code: "cancelled" }); + expect(uploadChunk).toHaveBeenCalledTimes(2); + expect(api.media.uploads.complete).not.toHaveBeenCalled(); + await expect(readdir(tempRoot)).resolves.toEqual([]); + }); +}); + +describe("Apple Photos sync state file", () => { + it("keeps a stable device id across launches", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "rakazo-photos-state-")); + try { + const store = createPhotosSyncFileStore(dir, () => "device-a"); + const first = await store.read(); + expect(first.deviceId).toBe("device-a"); + first.targets.x = { sourceId: "s", listing: null, changeToken: "t", uploads: {} }; + await store.write(first); + const reopened = await createPhotosSyncFileStore(dir, () => "device-b").read(); + expect(reopened).toEqual(first); + expect(await readFile(path.join(dir, "media-photos-sync.json"), "utf8")).toContain( + "device-a", + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it("drops unreadable state and malformed targets", () => { + expect(parsePhotosSyncState("{")).toBeNull(); + expect( + parsePhotosSyncState(JSON.stringify({ version: 2, deviceId: "d", targets: {} })), + ).toBeNull(); + expect( + parsePhotosSyncState( + JSON.stringify({ + version: 1, + deviceId: "d", + targets: { + good: { sourceId: "s", listing: null, changeToken: null, uploads: {} }, + bad: { sourceId: 4, listing: null, changeToken: null, uploads: {} }, + badListing: { sourceId: "s", listing: { cursor: 3 }, changeToken: null, uploads: {} }, + }, + }), + )?.targets, + ).toEqual({ good: { sourceId: "s", listing: null, changeToken: null, uploads: {} } }); + }); +}); + +describe("chunk uploader", () => { + it("PUTs raw bytes with the session and space", async () => { + const fetch = vi.fn(async () => new Response(null, { status: 204 })); + const upload = createChunkUploader({ + origin: "https://rakazo.example.com", + spaceId: "space_1", + fetch, + }); + const signal = new AbortController().signal; + await upload("up/1", 3, new Uint8Array([1, 2, 3]), signal); + expect(fetch).toHaveBeenCalledWith( + "https://rakazo.example.com/api/media/uploads/up%2F1/chunks/3", + expect.objectContaining({ + method: "PUT", + credentials: "include", + redirect: "error", + signal, + headers: { "content-type": "application/octet-stream", "x-rakazo-space-id": "space_1" }, + }), + ); + }); + + it("rejects failed chunks and non-origin targets", async () => { + const fetch = vi.fn(async () => new Response(null, { status: 409 })); + const upload = createChunkUploader({ + origin: "https://rakazo.example.com", + spaceId: null, + fetch, + }); + await expect( + upload("u", 0, new Uint8Array(), new AbortController().signal), + ).rejects.toMatchObject({ + code: "upload_failed", + }); + expect(() => + createChunkUploader({ origin: "https://rakazo.example.com/rpc", spaceId: null, fetch }), + ).toThrow(); + expect(() => createChunkUploader({ origin: "file:///etc", spaceId: null, fetch })).toThrow(); + }); +}); + +describe("runPool", () => { + it("bounds concurrency and reports the first failure", async () => { + let active = 0; + let max = 0; + const seen: number[] = []; + await expect( + runPool([1, 2, 3, 4, 5], 2, new AbortController().signal, async (item, signal) => { + active += 1; + max = Math.max(max, active); + await new Promise((resolve) => setTimeout(resolve, 5)); + active -= 1; + if (item === 2) throw new Error("boom"); + signal.throwIfAborted(); + seen.push(item); + }), + ).rejects.toThrow("boom"); + expect(max).toBe(2); + expect(seen).not.toContain(4); + }); +}); diff --git a/apps/desktop/src/media-photos-sync.ts b/apps/desktop/src/media-photos-sync.ts new file mode 100644 index 0000000..72ea3f9 --- /dev/null +++ b/apps/desktop/src/media-photos-sync.ts @@ -0,0 +1,590 @@ +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { mkdtemp, open, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { ClientMediaItem, MediaAsset, MediaUpload, mediaRpcShapes } from "@rakazo/contracts"; +import type { PhotosHelper } from "./media-photos-helper.js"; +import { PHOTOS_MAX_PAGE } from "./media-photos-helper.js"; +import { readPrivateFile, writePrivateFile } from "./setup-store.js"; + +/* + * Apple Photos is a client-pushed source: the API may run on another machine, so the + * app never sends a local path. It pushes metadata in batches and uploads the originals + * the server asks for (`awaiting_upload`) in resumable raw-byte chunks. + */ + +type Shapes = typeof mediaRpcShapes; +type Shape = { input: { _input: unknown }; output: { _output: unknown } }; +/** One oRPC procedure as `ContractRouterClient` exposes it. */ +type Procedure = ( + input: S["input"]["_input"], + options?: { signal?: AbortSignal }, +) => Promise; + +/** The slice of the app RPC client the sync needs; `rpc` from a contract client satisfies it. */ +export interface MediaSyncApi { + media: { + sources: { + registerClient: Procedure; + pushItems: Procedure; + }; + assets: { search: Procedure }; + uploads: { + begin: Procedure; + status: Procedure; + complete: Procedure; + }; + }; +} + +/** Sends one raw chunk to `PUT /api/media/uploads/:uploadId/chunks/:index`. */ +export type UploadChunk = ( + uploadId: string, + index: number, + bytes: Uint8Array, + signal: AbortSignal, +) => Promise; + +export type PhotosSyncErrorCode = "not_authorized" | "cancelled" | "integrity" | "upload_failed"; + +export class PhotosSyncError extends Error { + constructor( + readonly code: PhotosSyncErrorCode, + message: string, + ) { + super(message); + this.name = "PhotosSyncError"; + } +} + +// --------------------------------------------------------------------------- +// Persistent state +// --------------------------------------------------------------------------- + +export interface PhotosSyncTargetState { + sourceId: string; + /** A full listing in progress: where to resume, the token from its start and the last pushed item. */ + listing: { cursor: string | null; token: string | null; lastItem: ClientMediaItem | null } | null; + /** Set after a complete listing; later syncs only read changes since it. */ + changeToken: string | null; + /** Open uploads by asset id so an interrupted upload resumes instead of restarting. */ + uploads: Record; +} + +export interface PhotosSyncState { + version: 1; + /** Stable per installation; the server keys the Apple Photos source by it. */ + deviceId: string; + /** Keyed by server origin and space, since each has its own source. */ + targets: Record; +} + +export interface PhotosSyncStore { + read(): Promise; + write(state: PhotosSyncState): Promise; +} + +export const PHOTOS_SYNC_FILE_NAME = "media-photos-sync.json"; +const MAX_STATE_BYTES = 4 * 1024 * 1024; + +/** Kept next to the desktop setup file, private to the user and written atomically. */ +export function createPhotosSyncFileStore( + userDataDir: string, + newDeviceId: () => string = randomUUID, +): PhotosSyncStore { + const file = path.join(userDataDir, PHOTOS_SYNC_FILE_NAME); + return { + async read() { + const raw = await readPrivateFile(file, MAX_STATE_BYTES); + return (raw === null ? null : parsePhotosSyncState(raw)) ?? emptyState(newDeviceId()); + }, + write: (state) => writePrivateFile(file, `${JSON.stringify(state)}\n`), + }; +} + +function emptyState(deviceId: string): PhotosSyncState { + return { version: 1, deviceId, targets: {} }; +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); +const isNullableString = (value: unknown) => value === null || typeof value === "string"; + +/** Unreadable state starts over: the server deduplicates by device and external id. */ +export function parsePhotosSyncState(raw: string): PhotosSyncState | null { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + return null; + } + if ( + !isRecord(value) || + value.version !== 1 || + typeof value.deviceId !== "string" || + value.deviceId === "" || + value.deviceId.length > 200 || + !isRecord(value.targets) + ) { + return null; + } + const targets: Record = {}; + for (const [key, target] of Object.entries(value.targets)) { + if ( + !isRecord(target) || + typeof target.sourceId !== "string" || + !isNullableString(target.changeToken) || + !isRecord(target.uploads) || + !Object.values(target.uploads).every((id) => typeof id === "string") + ) { + continue; + } + const listing = target.listing; + const validListing = + listing === null || + (isRecord(listing) && + isNullableString(listing.cursor) && + isNullableString(listing.token) && + (listing.lastItem === null || isRecord(listing.lastItem))); + if (!validListing) continue; + targets[key] = target as unknown as PhotosSyncTargetState; + } + return { version: 1, deviceId: value.deviceId, targets }; +} + +// --------------------------------------------------------------------------- +// Chunk upload over HTTP +// --------------------------------------------------------------------------- + +/** + * Chunk PUTs ride the same signed-in session as the app window: the injected `fetch` + * must carry that session's cookies (Electron `session.fetch` with credentials). + */ +export function createChunkUploader(options: { + origin: string; + spaceId: string | null; + fetch: (url: string, init: RequestInit) => Promise; +}): UploadChunk { + const origin = new URL(options.origin); + if (!["http:", "https:"].includes(origin.protocol) || origin.origin !== options.origin) { + throw new Error("Uploads need the server origin."); + } + return async (uploadId, index, bytes, signal) => { + if (!Number.isSafeInteger(index) || index < 0) throw new Error("Invalid chunk index."); + const headers: Record = { "content-type": "application/octet-stream" }; + if (options.spaceId) headers["x-rakazo-space-id"] = options.spaceId; + const response = await options.fetch( + `${origin.origin}/api/media/uploads/${encodeURIComponent(uploadId)}/chunks/${index}`, + { + method: "PUT", + headers, + body: bytes, + credentials: "include", + redirect: "error", + signal, + }, + ); + if (!response.ok) { + throw new PhotosSyncError( + "upload_failed", + `The server rejected an upload (${response.status}).`, + ); + } + }; +} + +// --------------------------------------------------------------------------- +// Sync +// --------------------------------------------------------------------------- + +export interface PhotosSyncProgress { + phase: "authorizing" | "indexing" | "uploading"; + /** Items pushed in this run. */ + indexed: number; + uploadsDone: number; + uploadsTotal: number; +} + +export interface PhotosSyncResult { + indexed: number; + created: number; + updated: number; + unchanged: number; + uploaded: number; + /** True when this run listed the whole library instead of reading changes. */ + fullListing: boolean; +} + +export interface PhotosSyncOptions { + helper: PhotosHelper; + api: MediaSyncApi; + uploadChunk: UploadChunk; + store: PhotosSyncStore; + /** Server origin plus space, e.g. `https://rakazo.example.com#space_1`. */ + target: string; + /** Source label shown in the library, e.g. the Mac's name or "Apple Photos". */ + label: string; + /** Exports land in a fresh folder below this and are deleted after upload. */ + tempRoot: string; + signal: AbortSignal; + onProgress?: (progress: PhotosSyncProgress) => void; + /** At most 500, the contract's batch limit. */ + batchSize?: number; + concurrency?: number; +} + +const MAX_SYNC_TOKEN_LENGTH = 2000; + +export async function syncApplePhotos(options: PhotosSyncOptions): Promise { + const { helper, api, signal } = options; + const batchSize = Math.min(Math.max(1, options.batchSize ?? PHOTOS_MAX_PAGE), PHOTOS_MAX_PAGE); + const result: PhotosSyncResult = { + indexed: 0, + created: 0, + updated: 0, + unchanged: 0, + uploaded: 0, + fullListing: false, + }; + const progress = { uploadsDone: 0, uploadsTotal: 0 }; + const report = (phase: PhotosSyncProgress["phase"]) => + options.onProgress?.({ phase, indexed: result.indexed, ...progress }); + + const state = await options.store.read(); + // Writes are serialized so concurrent upload workers never interleave partial state. + let saving = Promise.resolve(); + const save = () => { + const snapshot = JSON.parse(JSON.stringify(state)) as PhotosSyncState; + saving = saving.then(() => options.store.write(snapshot)); + return saving; + }; + + try { + report("authorizing"); + const authorization = await helper.authorize(true, signal); + if (authorization !== "authorized" && authorization !== "limited") { + throw new PhotosSyncError("not_authorized", "Photos access was not granted."); + } + + const ensureTarget = async (): Promise => { + const existing = state.targets[options.target]; + if (existing) return existing; + const source = await api.media.sources.registerClient( + { kind: "apple_photos", label: options.label, deviceId: state.deviceId }, + { signal }, + ); + const created: PhotosSyncTargetState = { + sourceId: source.id, + listing: null, + changeToken: null, + uploads: {}, + }; + state.targets[options.target] = created; + await save(); + return created; + }; + + const push = async ( + target: PhotosSyncTargetState, + items: ClientMediaItem[], + syncToken: string | null, + complete: boolean, + deletedExternalIds: string[] = [], + ) => { + const counts = await api.media.sources.pushItems( + { + sourceId: target.sourceId, + items, + deletedExternalIds, + syncToken: + syncToken !== null && syncToken.length <= MAX_SYNC_TOKEN_LENGTH ? syncToken : null, + complete, + }, + { signal }, + ); + result.indexed += items.length; + result.created += counts.created; + result.updated += counts.updated; + result.unchanged += counts.unchanged; + report("indexing"); + }; + + /** Full listing, resumable page by page; the last push says `complete`. */ + const listAll = async (target: PhotosSyncTargetState) => { + result.fullListing = true; + target.listing ??= { cursor: null, token: null, lastItem: null }; + const listing = target.listing; + while (true) { + const page = await helper.list({ cursor: listing.cursor, limit: batchSize }, signal); + if (listing.cursor === null) listing.token = page.changeToken; + const done = page.nextCursor === null; + const items = page.items; + // An empty final page still closes the listing. + if (items.length > 0 || done) await push(target, items, done ? listing.token : null, done); + if (done) { + target.changeToken = listing.token; + target.listing = null; + await save(); + return; + } + listing.cursor = page.nextCursor; + listing.lastItem = items.at(-1) ?? listing.lastItem; + await save(); + } + }; + + const syncMetadata = async () => { + const target = await ensureTarget(); + if (target.listing !== null || target.changeToken === null) { + await listAll(target); + return; + } + const changes = await helper.changes(target.changeToken, signal); + if (changes.fullResyncRequired || changes.token === null) { + target.listing = null; + await listAll(target); + return; + } + const batches = Math.max( + Math.ceil(changes.changed.length / batchSize), + Math.ceil(changes.deleted.length / DELETED_BATCH_SIZE), + ); + for (let index = 0; index < batches; index++) { + const batch = changes.changed.slice(index * batchSize, (index + 1) * batchSize); + const deleted = changes.deleted.slice( + index * DELETED_BATCH_SIZE, + (index + 1) * DELETED_BATCH_SIZE, + ); + const last = index === batches - 1; + await push(target, batch, last ? changes.token : null, false, deleted); + } + target.changeToken = changes.token; + await save(); + }; + + report("indexing"); + try { + await syncMetadata(); + } catch (error) { + // The source was removed on the server: register again and list from scratch. + if (!isNotFound(error) || signal.aborted) throw error; + delete state.targets[options.target]; + await save(); + await syncMetadata(); + } + + const target = state.targets[options.target]!; + const awaiting = await awaitingUploads(api, target.sourceId, signal); + progress.uploadsTotal = awaiting.length; + // Forget uploads for assets the server no longer waits for. + const waiting = new Set(awaiting.map((asset) => asset.id)); + for (const assetId of Object.keys(target.uploads)) { + if (!waiting.has(assetId)) delete target.uploads[assetId]; + } + await save(); + report("uploading"); + await runPool(awaiting, options.concurrency ?? 2, signal, async (asset, workerSignal) => { + await uploadAsset({ ...options, signal: workerSignal }, target, asset, save); + result.uploaded += 1; + progress.uploadsDone += 1; + report("uploading"); + }); + return result; + } catch (error) { + if (signal.aborted) throw new PhotosSyncError("cancelled", "Sync cancelled."); + throw error; + } finally { + await saving.catch(() => undefined); + } +} + +/** + * Binds the sync to the server and space the request names. `connect` builds the RPC + * client and chunk uploader on the signed-in session for that space. + */ +export function createPhotosSyncRunner(deps: { + helper: PhotosHelper; + store: PhotosSyncStore; + tempRoot: string; + label: string; + connect: ( + spaceId: string | null, + ) => Promise<{ origin: string; api: MediaSyncApi; uploadChunk: UploadChunk }>; +}) { + return async (request: { + spaceId: string | null; + signal: AbortSignal; + onProgress: (progress: PhotosSyncProgress) => void; + }): Promise => { + const { origin, api, uploadChunk } = await deps.connect(request.spaceId); + return syncApplePhotos({ + helper: deps.helper, + store: deps.store, + tempRoot: deps.tempRoot, + label: deps.label, + api, + uploadChunk, + target: `${origin}#${request.spaceId ?? "default"}`, + signal: request.signal, + onProgress: request.onProgress, + }); + }; +} + +function isNotFound(error: unknown): boolean { + return isRecord(error) && error.code === "NOT_FOUND"; +} + +interface AwaitingAsset { + id: string; + externalId: string; +} + +/** Assets of this source the server is waiting for, mapped back to Photos items. */ +const DELETED_BATCH_SIZE = 5000; + +export async function awaitingUploads( + api: MediaSyncApi, + sourceId: string, + signal: AbortSignal, +): Promise { + const awaiting: AwaitingAsset[] = []; + let cursor: string | null = null; + do { + const page: { items: MediaAsset[]; nextCursor: string | null } = await api.media.assets.search( + { sourceId, ingestStates: ["awaiting_upload"], cursor, limit: PHOTOS_MAX_PAGE }, + { signal }, + ); + for (const asset of page.items) { + if (asset.ingestState !== "awaiting_upload" || asset.sourceId !== sourceId) continue; + awaiting.push({ id: asset.id, externalId: asset.externalId }); + } + cursor = page.nextCursor; + } while (cursor !== null); + return awaiting; +} + +async function uploadAsset( + options: PhotosSyncOptions, + target: PhotosSyncTargetState, + asset: AwaitingAsset, + save: () => Promise, +): Promise { + const { api, signal } = options; + const directory = await mkdtemp(path.join(options.tempRoot, "rakazo-photos-")); + try { + const exported = await options.helper.exportOriginal(asset.externalId, directory, { signal }); + // The helper's hash is checked against the bytes on disk before anything is sent. + const local = await hashFile(exported.path, signal); + if (local.sha256 !== exported.sha256 || local.sizeBytes !== exported.sizeBytes) { + throw new PhotosSyncError("integrity", "The exported file does not match its checksum."); + } + + let upload: MediaUpload | null = null; + const previous = target.uploads[asset.id]; + if (previous) { + const status = await api.media.uploads + .status({ uploadId: previous }, { signal }) + .catch((error: unknown) => { + if (isNotFound(error)) return null; + throw error; + }); + if (status?.status === "complete") { + delete target.uploads[asset.id]; + await save(); + return; + } + if (status?.status === "open" && status.sizeBytes === local.sizeBytes) upload = status; + } + if (upload === null) { + upload = await api.media.uploads.begin( + { + assetId: asset.id, + sizeBytes: local.sizeBytes, + sha256: local.sha256, + fileName: exported.fileName.slice(0, 255), + }, + { signal }, + ); + target.uploads[asset.id] = upload.id; + await save(); + } + + const received = new Set(upload.receivedChunks); + const chunkCount = Math.ceil(local.sizeBytes / upload.chunkBytes); + const file = await open(exported.path, "r"); + try { + for (let index = 0; index < chunkCount; index += 1) { + if (received.has(index)) continue; + signal.throwIfAborted(); + const offset = index * upload.chunkBytes; + const length = Math.min(upload.chunkBytes, local.sizeBytes - offset); + const bytes = Buffer.alloc(length); + const { bytesRead } = await file.read(bytes, 0, length, offset); + if (bytesRead !== length) { + throw new PhotosSyncError("integrity", "The exported file changed during upload."); + } + await options.uploadChunk(upload.id, index, bytes, signal); + } + } finally { + await file.close(); + } + const completed = await api.media.uploads.complete({ uploadId: upload.id }, { signal }); + if (completed.status !== "complete") { + throw new PhotosSyncError("upload_failed", "The server did not accept the upload."); + } + delete target.uploads[asset.id]; + await save(); + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +export async function hashFile( + file: string, + signal?: AbortSignal, +): Promise<{ sha256: string; sizeBytes: number }> { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(file, { signal })) hash.update(chunk as Buffer); + return { sha256: hash.digest("hex"), sizeBytes: (await stat(file)).size }; +} + +/** + * Runs `work` with at most `limit` items in flight. The first failure aborts the + * others and is the error reported. + */ +export async function runPool( + items: T[], + limit: number, + signal: AbortSignal, + work: (item: T, signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + const forward = () => controller.abort(signal.reason); + signal.addEventListener("abort", forward, { once: true }); + if (signal.aborted) forward(); + let next = 0; + let failed = false; + let failure: unknown; + const worker = async () => { + while (!failed && !controller.signal.aborted && next < items.length) { + const item = items[next++]!; + try { + await work(item, controller.signal); + } catch (error) { + if (!failed) { + failed = true; + failure = error; + controller.abort(error); + } + } + } + }; + try { + await Promise.all(Array.from({ length: Math.min(Math.max(1, limit), items.length) }, worker)); + } finally { + signal.removeEventListener("abort", forward); + } + if (failed) throw failure; + signal.throwIfAborted(); +} diff --git a/apps/mobile/package.json b/apps/mobile/package.json index d5bb46b..176c162 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -35,6 +35,7 @@ "expo-image-picker": "57.0.20", "expo-linking": "57.0.11", "expo-localization": "57.0.2", + "expo-media-library": "57.0.5", "expo-modules-core": "57.0.20", "expo-notifications": "57.0.21", "expo-router": "57.0.24", @@ -44,6 +45,7 @@ "expo-status-bar": "57.0.1", "expo-symbols": "57.0.3", "expo-updates": "57.0.24", + "expo-video": "57.0.5", "react": "19.2.3", "react-dom": "19.2.3", "react-native": "0.86.3", diff --git a/apps/web/package.json b/apps/web/package.json index 1f121dc..a9326b8 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -24,6 +24,7 @@ "@rakazo/core": "workspace:*", "@rakazo/ui-tokens": "workspace:*", "@rakazo/ui-web": "workspace:*", + "@tanstack/react-virtual": "3.14.13", "better-auth": "1.7.4", "lucide-react": "1.44.0", "react": "19.2.3", diff --git a/packages/adapter-kit/src/background-jobs.test.ts b/packages/adapter-kit/src/background-jobs.test.ts index 13880d7..df0f258 100644 --- a/packages/adapter-kit/src/background-jobs.test.ts +++ b/packages/adapter-kit/src/background-jobs.test.ts @@ -22,6 +22,12 @@ function handlers(): BackgroundJobHandlers { "messaging.deliver": vi.fn(async () => undefined), "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), + "media.sync": vi.fn(async () => undefined), + "media.proxy": vi.fn(async () => undefined), + "media.analyze": vi.fn(async () => undefined), + "media.cluster": vi.fn(async () => undefined), + "media.render": vi.fn(async () => undefined), + "media.publish": vi.fn(async () => undefined), }; } diff --git a/packages/adapter-kit/src/background-jobs.ts b/packages/adapter-kit/src/background-jobs.ts index 45cad56..3f09390 100644 --- a/packages/adapter-kit/src/background-jobs.ts +++ b/packages/adapter-kit/src/background-jobs.ts @@ -1,3 +1,5 @@ +import type { MediaAnalysisKind } from "@rakazo/contracts"; +import { MediaAnalysisKindSchema } from "@rakazo/contracts"; import { z } from "zod"; import type { BackgroundJob, @@ -23,6 +25,15 @@ const payloadSchemas = { "messaging.deliver": z.object({ runId: z.string().min(1).optional() }), "cloud_agent.poll": z.object({ agentId: z.string().min(1) }), "profile.consolidate": z.object({ userId: z.string().min(1) }), + "media.sync": z.object({ sourceId: z.string().min(1) }), + "media.proxy": z.object({ assetId: z.string().min(1) }), + "media.analyze": z.object({ + assetId: z.string().min(1), + kinds: z.array(MediaAnalysisKindSchema).optional(), + }), + "media.cluster": z.object({ spaceId: z.string().min(1) }), + "media.render": z.object({ renderId: z.string().min(1) }), + "media.publish": z.object({ publicationId: z.string().min(1) }), } satisfies { [Name in BackgroundJobName]: z.ZodType }; export function parseBackgroundJob(name: string, payload: unknown): BackgroundJob { @@ -160,3 +171,41 @@ export function profileConsolidateJob(userId: string): BackgroundJob { replaceKey: profileConsolidateJobKey(userId), }; } + +export function mediaSyncJob(sourceId: string): BackgroundJob { + return { name: "media.sync", payload: { sourceId }, replaceKey: `media.sync:${sourceId}` }; +} + +export function mediaProxyJob(assetId: string): BackgroundJob { + return { name: "media.proxy", payload: { assetId }, replaceKey: `media.proxy:${assetId}` }; +} + +export function mediaAnalyzeJob(assetId: string, kinds?: MediaAnalysisKind[]): BackgroundJob { + return { + name: "media.analyze", + payload: kinds ? { assetId, kinds } : { assetId }, + replaceKey: `media.analyze:${assetId}`, + }; +} + +export function mediaClusterJob(spaceId: string, availableAt?: Date): BackgroundJob { + return { + name: "media.cluster", + payload: { spaceId }, + replaceKey: `media.cluster:${spaceId}`, + ...(availableAt ? { availableAt } : {}), + }; +} + +export function mediaRenderJob(renderId: string): BackgroundJob { + return { name: "media.render", payload: { renderId }, replaceKey: `media.render:${renderId}` }; +} + +export function mediaPublishJob(publicationId: string, availableAt?: Date): BackgroundJob { + return { + name: "media.publish", + payload: { publicationId }, + replaceKey: `media.publish:${publicationId}`, + ...(availableAt ? { availableAt } : {}), + }; +} diff --git a/packages/adapter-kit/src/index.ts b/packages/adapter-kit/src/index.ts index 43f7898..e39a95c 100644 --- a/packages/adapter-kit/src/index.ts +++ b/packages/adapter-kit/src/index.ts @@ -1,5 +1,6 @@ export * from "./background-jobs.js"; export * from "./cloud-agents.js"; export * from "./interfaces.js"; +export * from "./media.js"; export * from "./registry.js"; export * from "./types.js"; diff --git a/packages/adapter-kit/src/media.ts b/packages/adapter-kit/src/media.ts new file mode 100644 index 0000000..1674304 --- /dev/null +++ b/packages/adapter-kit/src/media.ts @@ -0,0 +1,325 @@ +import type { + AnonymizeMode, + BoxKeyframe, + BrandKit, + ContentCategory, + FfprobeReport, + MediaAssetKind, + MediaCapabilities, + MediaInferenceOperation, + MediaMarkPrompt, + MediaSourceKind, + MotionTemplateInfo, + MotionTemplateRef, + NormalizedBox, + NormalizedPoint, + PlatformProfile, + RenderPlan, +} from "@rakazo/contracts"; + +/* + * Media Studio provider boundaries. Every implementation (local sidecar, + * emulator, optional cloud adapters) satisfies the same contracts, which + * media-conformance tests exercise offline. + * + * Paths are absolute host paths inside the configured media root. Providers + * must reject any path outside that root. + */ + +// --------------------------------------------------------------------------- +// Sources +// --------------------------------------------------------------------------- + +export interface MediaSourceItem { + externalId: string; + kind: MediaAssetKind; + capturedAt: Date | null; + durationMs: number | null; + width: number | null; + height: number | null; + fps: number | null; + gpsLat: number | null; + gpsLon: number | null; + deviceModel: string | null; + albumNames: string[]; + isFavorite: boolean; + fileName: string | null; + sizeBytes: number | null; + /** Cheap change detector (mtime + size or provider revision); not a content hash. */ + revision: string; +} + +export interface MediaSourceListRequest { + config: Record; + cursor: string | null; + limit: number; + signal?: AbortSignal; +} + +export interface MediaSourceListResult { + items: MediaSourceItem[]; + nextCursor: string | null; +} + +/** Server-side sources the worker can enumerate and read directly. */ +export interface MediaSourceProvider { + readonly kind: Extract; + list(request: MediaSourceListRequest): Promise; + /** Absolute readable path of the original file. Throws when it escaped the source root. */ + resolve(config: Record, externalId: string): Promise; +} + +// --------------------------------------------------------------------------- +// Inference +// --------------------------------------------------------------------------- + +export class UnsupportedMediaOperationError extends Error { + constructor( + readonly operation: MediaInferenceOperation, + readonly reason: string, + ) { + super(`${operation} is not available: ${reason}`); + this.name = "UnsupportedMediaOperationError"; + } +} + +export interface MediaTimeSpan { + startMs: number; + endMs: number; +} + +export interface DetectScenesResult { + shots: Array; + scenes: MediaTimeSpan[]; +} + +export interface TranscriptWord { + startMs: number; + endMs: number; + text: string; + confidence: number; +} + +export interface TranscribeResult { + language: string | null; + words: TranscriptWord[]; +} + +export interface FaceObservation { + box: NormalizedBox; + confidence: number; + /** Five-point landmarks: eyes, nose, mouth corners. */ + landmarks?: NormalizedPoint[]; +} + +export interface DetectFacesResult { + frames: Array<{ tMs: number; faces: FaceObservation[] }>; +} + +export interface FaceTrackResult { + trackId: string; + keyframes: Array; + /** Identity embedding for local person clustering. Never leaves the host. */ + embedding: number[] | null; +} + +export interface TrackFacesResult { + sampleFps: number; + tracks: FaceTrackResult[]; +} + +export interface FaceLandmarksResult { + frames: Array<{ + tMs: number; + faces: Array<{ box: NormalizedBox; landmarks: NormalizedPoint[] }>; + }>; +} + +export interface EmbedFramesResult { + model: string; + dim: number; + vectors: number[][]; +} + +export interface ClassifyContentResult { + model: string; + frames: Array<{ tMs: number; scores: Partial> }>; +} + +export interface SegmentPromptedResult { + /** Directory of per-frame PNG masks named by timestamp in ms. */ + maskDir: string; + keyframes: Array; + score: number; +} + +export interface InpaintQuality { + /** 1 = no flicker inside the mask region between neighbouring frames. */ + temporalStability: number; + /** 1 = no visible seams at the mask edge. */ + edgeQuality: number; + score: number; +} + +export interface InpaintVideoResult { + outPath: string; + quality: InpaintQuality; +} + +export interface AudioFeaturesResult { + integratedLufs: number | null; + truePeakDb: number | null; + tempoBpm: number | null; + beatsMs: number[]; + energy: Array<{ tMs: number; value: number }>; + silences: MediaTimeSpan[]; +} + +export interface SaliencyResult { + points: Array<{ tMs: number; cx: number; cy: number; strength: number }>; +} + +export interface RedactionInstruction { + keyframes: BoxKeyframe[]; + mode: Exclude | "cover"; + paddingPct: number; + featherPct: number; + /** Solid and cover fill. */ + color?: string; + /** Landmarks per keyframe for partial modes. */ + landmarks?: Array<{ tMs: number; points: NormalizedPoint[] }>; + /** Mask directory for region marks instead of boxes. */ + maskDir?: string; +} + +interface MediaPathRequest { + path: string; + signal?: AbortSignal; +} + +/** + * Local-first ML operations. Callers check `capabilities()` and fall back + * (blur, cover, reframe) when an operation reports unavailable; providers + * throw UnsupportedMediaOperationError instead of guessing. + */ +export interface MediaInferenceProvider { + readonly id: string; + capabilities(signal?: AbortSignal): Promise; + detectScenes(request: MediaPathRequest): Promise; + transcribe(request: MediaPathRequest & { language?: string }): Promise; + detectFaces( + request: MediaPathRequest & { sampleFps: number; timesMs?: number[] }, + ): Promise; + trackFaces(request: MediaPathRequest & { sampleFps: number }): Promise; + faceLandmarks(request: MediaPathRequest & { timesMs: number[] }): Promise; + embedFrames(request: MediaPathRequest & { timesMs: number[] }): Promise; + classifyContent( + request: MediaPathRequest & { sampleFps: number; extraTimesMs?: number[] }, + ): Promise; + segmentPrompted( + request: MediaPathRequest & { prompt: MediaMarkPrompt; propagate: boolean; outDir: string }, + ): Promise; + inpaintVideo( + request: MediaPathRequest & { maskDir: string; outPath: string }, + ): Promise; + /** + * Apply face and region redactions frame-accurately and write a near-lossless + * intermediate. Fails closed: a frame it cannot process is an error, not a pass. + */ + redact( + request: MediaPathRequest & { redactions: RedactionInstruction[]; outPath: string }, + ): Promise<{ outPath: string; framesProcessed: number }>; + /** + * Replace a tracked face with a generated, non-existent identity derived from + * `seed`. Deliberately takes no target-face image. + */ + synthesizeIdentity( + request: MediaPathRequest & { track: BoxKeyframe[]; seed: number; outPath: string }, + ): Promise<{ outPath: string }>; + audioFeatures(request: MediaPathRequest): Promise; + saliency(request: MediaPathRequest & { timesMs: number[] }): Promise; +} + +// --------------------------------------------------------------------------- +// Rendering +// --------------------------------------------------------------------------- + +export interface MediaRenderProgress { + stepId: string; + /** Overall progress 0..1. */ + progress: number; +} + +export interface MediaRenderer { + readonly id: string; + /** Hardware encoders usable on this host, e.g. h264_vaapi, h264_videotoolbox. */ + hardwareEncoders(): Promise; + /** + * Run every ffmpeg step of the plan. Motion, inpaint, and synthetic identity + * steps must already be satisfied (their outPath exists) or the renderer throws. + */ + render( + plan: RenderPlan, + options: { signal?: AbortSignal; onProgress?: (progress: MediaRenderProgress) => void }, + ): Promise; + probe( + path: string, + options?: { measureLoudness?: boolean; signal?: AbortSignal }, + ): Promise; + /** Extract a still frame as PNG (EXIF-free). */ + extractFrame(input: { + path: string; + tMs: number; + outPath: string; + width?: number; + signal?: AbortSignal; + }): Promise; +} + +export interface MotionRenderRequest { + template: MotionTemplateRef; + brandKit: BrandKit; + /** Resolved font and logo files from the brand kit artifacts. */ + assets: { fonts: Array<{ family: string; path: string }>; logoPaths: string[] }; + width: number; + height: number; + fps: number; + outPath: string; + signal?: AbortSignal; +} + +export interface MotionGraphicsRenderer { + readonly id: string; + templates(): MotionTemplateInfo[]; + /** Render an overlay clip with alpha (MOV, qtrle or prores 4444). */ + render(request: MotionRenderRequest): Promise<{ path: string; hasAlpha: boolean }>; +} + +// --------------------------------------------------------------------------- +// Publishing +// --------------------------------------------------------------------------- + +export interface MediaPublishRequest { + bundleDir: string; + profile: PlatformProfile; + publicationId: string; + title: string | null; + caption: string | null; + tags: string[]; + signal?: AbortSignal; +} + +export interface MediaPublishResult { + status: "published" | "bundled"; + externalUrl: string | null; +} + +export interface MediaPublisher { + readonly id: string; + publish(request: MediaPublishRequest): Promise; +} + +export interface LinkShortener { + readonly id: string; + shorten(url: string, signal?: AbortSignal): Promise; +} diff --git a/packages/adapter-kit/src/registry.ts b/packages/adapter-kit/src/registry.ts index 19c0822..086efa2 100644 --- a/packages/adapter-kit/src/registry.ts +++ b/packages/adapter-kit/src/registry.ts @@ -36,4 +36,8 @@ export const slots = { web: "web", browser: "browser", cloudAgent: "cloud-agent", + mediaInference: "media-inference", + mediaRenderer: "media-renderer", + motionGraphics: "motion-graphics", + mediaPublisher: "media-publisher", } as const; diff --git a/packages/adapter-kit/src/types.ts b/packages/adapter-kit/src/types.ts index 0c480ac..5bbfb43 100644 --- a/packages/adapter-kit/src/types.ts +++ b/packages/adapter-kit/src/types.ts @@ -1,4 +1,9 @@ -import type { ConnectionCatalogItem, ModelTier, SandboxKind } from "@rakazo/contracts"; +import type { + ConnectionCatalogItem, + MediaAnalysisKind, + ModelTier, + SandboxKind, +} from "@rakazo/contracts"; export interface AdapterContext { operationId: string; @@ -524,9 +529,22 @@ export interface BackgroundJobPayloads { "cloud_agent.poll": { agentId: string }; /** Learn durable user facts from conversations since the profile's last consolidation. */ "profile.consolidate": { userId: string }; + /** Enumerate a server-side media source incrementally (metadata only). */ + "media.sync": { sourceId: string }; + /** Hash the original and build proxy, thumbnails, and keyframes. */ + "media.proxy": { assetId: string }; + /** Run sidecar analyses; omitted kinds means every analysis not yet at the current version. */ + "media.analyze": { assetId: string; kinds?: MediaAnalysisKind[] }; + /** Rebuild sessions, themes, and person suggestions for a space. */ + "media.cluster": { spaceId: string }; + /** Render, probe, and gate one MediaRender. */ + "media.render": { renderId: string }; + /** Build the export bundle and publish a scheduled MediaPublication. */ + "media.publish": { publicationId: string }; } export type BackgroundJobName = keyof BackgroundJobPayloads; +export type MediaJobName = Extract; export type BackgroundJob = { [Name in BackgroundJobName]: { @@ -541,6 +559,8 @@ export type BackgroundJobHandlers = { [Name in BackgroundJobName]: (payload: BackgroundJobPayloads[Name]) => Promise; }; +export type MediaJobHandlers = Pick; + export interface SecretRecord { id: string; ciphertext: string; diff --git a/packages/adapters/package.json b/packages/adapters/package.json index c9b7855..d2e461d 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -29,6 +29,8 @@ "@rakazo/core": "workspace:*", "@rakazo/db": "workspace:*", "@rakazo/logging": "workspace:*", + "@rakazo/media": "workspace:*", + "@resvg/resvg-js": "2.6.2", "chat": "4.40.0", "chat-adapter-lark": "0.3.1", "chat-adapter-sendblue": "0.2.0", diff --git a/packages/adapters/src/background-job-handlers.ts b/packages/adapters/src/background-job-handlers.ts index af869ac..949c898 100644 --- a/packages/adapters/src/background-job-handlers.ts +++ b/packages/adapters/src/background-job-handlers.ts @@ -3,6 +3,7 @@ import type { AgentRuntime, BackgroundJobHandlers, JobPublisher, + MediaJobHandlers, MessagingSurface, SandboxProvider, } from "@rakazo/adapter-kit"; @@ -36,6 +37,8 @@ export function createBackgroundJobHandlers(deps: { deploymentModelKey?: string; messaging?: MessagingSurface; cloudAgent?: CloudAgentConnection | null; + /** Media Studio job handlers; absent when media is not configured for this worker. */ + media?: MediaJobHandlers; }): BackgroundJobHandlers { const deliverMessaging = async (runId?: string) => { if (!deps.messaging) return; @@ -53,6 +56,7 @@ export function createBackgroundJobHandlers(deps: { }; return { + ...(deps.media ?? mediaDisabledHandlers()), "run.continue": async (payload) => { await deps.executor.continueRun(payload.runId, deps.workerId); // Automatic messaging mirror: once the run's bot messages are durable, @@ -126,3 +130,17 @@ export function createBackgroundJobHandlers(deps: { }, }; } + +function mediaDisabledHandlers(): MediaJobHandlers { + const skip = async () => { + getLogger().warn("media job skipped: Media Studio is not configured on this worker"); + }; + return { + "media.sync": skip, + "media.proxy": skip, + "media.analyze": skip, + "media.cluster": skip, + "media.render": skip, + "media.publish": skip, + }; +} diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 7b4ecd0..f6a9368 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -76,6 +76,24 @@ export * from "./mcp-emulator.js"; export * from "./mcp-oauth.js"; export * from "./mcp-server-tool.js"; export * from "./mcp-transport.js"; +export * from "./media-context.js"; +export * from "./media-emulator.js"; +export * from "./media-emulator-clips.js"; +export * from "./media-emulator-source.js"; +export * from "./media-factory.js"; +export * from "./media-folder-source.js"; +export * from "./media-folder-source-probe.js"; +export type { + MotionFrameContext, + MotionTemplate, + ResvgMotionGraphicsRendererOptions, +} from "./media-motion-renderer.js"; +export { + MOTION_TEMPLATES, + MotionTemplateError, + ResvgMotionGraphicsRenderer, +} from "./media-motion-renderer.js"; +export * from "./media-sidecar.js"; export * from "./memory-provider-factory.js"; export * from "./messaging-context.js"; export * from "./messaging-delivery.js"; diff --git a/packages/adapters/src/media-conformance.test.ts b/packages/adapters/src/media-conformance.test.ts new file mode 100644 index 0000000..5d28ce6 --- /dev/null +++ b/packages/adapters/src/media-conformance.test.ts @@ -0,0 +1,281 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { MediaInferenceProvider } from "@rakazo/adapter-kit"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { MediaInferenceOperation } from "@rakazo/contracts"; +import { MediaCapabilitiesSchema, MediaInferenceOperationSchema } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { EmulatorMediaInferenceProvider } from "./media-emulator.js"; +import { extractRgbFrame, ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js"; +import { MEDIA_SIDECAR_RESPONSE_SCHEMAS, SidecarMediaInferenceProvider } from "./media-sidecar.js"; + +/* + * One contract for every MediaInferenceProvider. The emulator always runs; the + * sidecar runs only in the Docker integration job, which sets + * MEDIA_CONFORMANCE_URL, MEDIA_CONFORMANCE_TOKEN and MEDIA_CONFORMANCE_ROOT (a + * directory inside the sidecar's MEDIA_ROOT, mounted at the same path here). + */ + +const WIDTH = 320; +const HEIGHT = 180; +const hasFfmpeg = ffmpegAvailable(); + +interface Target { + name: string; + /** Directory inside the provider's root where the suite writes fixtures. */ + root: () => Promise; + provider: (root: string) => MediaInferenceProvider; +} + +const targets: Target[] = [ + { + name: "emulator", + root: () => mkdtemp(path.join(tmpdir(), "media-conformance-")), + provider: (root) => new EmulatorMediaInferenceProvider({ root }), + }, +]; + +const sidecarUrl = process.env.MEDIA_CONFORMANCE_URL; +const sidecarToken = process.env.MEDIA_CONFORMANCE_TOKEN; +const sidecarRoot = process.env.MEDIA_CONFORMANCE_ROOT; +if (sidecarUrl && sidecarToken && sidecarRoot) { + targets.push({ + name: "sidecar", + root: async () => { + await mkdir(sidecarRoot, { recursive: true }); + return mkdtemp(path.join(sidecarRoot, "conformance-")); + }, + provider: () => new SidecarMediaInferenceProvider({ baseUrl: sidecarUrl, token: sidecarToken }), + }); +} + +function requestFor(operation: MediaInferenceOperation, clip: string, root: string) { + const box = { x: 0.25, y: 0.25, w: 0.25, h: 0.25 }; + switch (operation) { + case "detectScenes": + case "audioFeatures": + return { path: clip }; + case "transcribe": + return { path: clip, language: "en" }; + case "detectFaces": + case "trackFaces": + return { path: clip, sampleFps: 2 }; + case "faceLandmarks": + case "embedFrames": + case "saliency": + return { path: clip, timesMs: [0, 500, 1000] }; + case "classifyContent": + return { path: clip, sampleFps: 2, extraTimesMs: [750] }; + case "segmentPrompted": + return { + path: clip, + prompt: { tMs: 500, box }, + propagate: true, + outDir: path.join(root, "masks"), + }; + case "inpaintVideo": + return { + path: clip, + maskDir: path.join(root, "masks"), + outPath: path.join(root, "inpaint.mp4"), + }; + case "redact": + return { + path: clip, + redactions: [ + { + keyframes: [ + { tMs: 0, box }, + { tMs: 2000, box }, + ], + mode: "blur" as const, + paddingPct: 0, + featherPct: 0, + }, + ], + outPath: path.join(root, "redacted-any.mp4"), + }; + case "synthesizeIdentity": + return { + path: clip, + track: [{ tMs: 0, box }], + seed: 7, + outPath: path.join(root, "synthetic.mp4"), + }; + } +} + +/** Mean absolute RGB difference inside and outside a pixel box (with a guard margin). */ +function regionDiff( + a: Buffer, + b: Buffer, + box: { x: number; y: number; w: number; h: number }, + margin = 4, +) { + let inside = 0; + let insideCount = 0; + let outside = 0; + let outsideCount = 0; + for (let y = 0; y < HEIGHT; y++) { + for (let x = 0; x < WIDTH; x++) { + const within = x >= box.x && x < box.x + box.w && y >= box.y && y < box.y + box.h; + const near = + x >= box.x - margin && + x < box.x + box.w + margin && + y >= box.y - margin && + y < box.y + box.h + margin; + let diff = 0; + for (let c = 0; c < 3; c++) { + const i = (y * WIDTH + x) * 3 + c; + diff += Math.abs((a[i] ?? 0) - (b[i] ?? 0)); + } + if (within) { + inside += diff / 3; + insideCount++; + } else if (!near) { + outside += diff / 3; + outsideCount++; + } + } + } + return { inside: inside / insideCount, outside: outside / outsideCount }; +} + +for (const target of targets) { + describe.skipIf(!hasFfmpeg)(`MediaInferenceProvider conformance: ${target.name}`, () => { + let root: string; + let clip: string; + let provider: MediaInferenceProvider; + + beforeAll(async () => { + root = await target.root(); + clip = await renderTestClip(path.join(root, "clip.mp4"), { + seconds: 2, + width: WIDTH, + height: HEIGHT, + }); + provider = target.provider(root); + }); + + afterAll(async () => { + if (root) await rm(root, { recursive: true, force: true }); + }); + + it("reports a capabilities document for every operation", async () => { + const capabilities = MediaCapabilitiesSchema.parse(await provider.capabilities()); + expect(Object.keys(capabilities.operations).sort()).toEqual( + [...MediaInferenceOperationSchema.options].sort(), + ); + }); + + it("returns schema-valid output or Unsupported for every operation", async () => { + const capabilities = await provider.capabilities(); + // segmentPrompted runs first so inpaintVideo finds its mask directory. + for (const operation of MediaInferenceOperationSchema.options) { + const request = requestFor(operation, clip, root); + const call = provider[operation] as (input: unknown) => Promise; + try { + const result = await call.call(provider, request); + const parsed = MEDIA_SIDECAR_RESPONSE_SCHEMAS[operation].safeParse(result); + expect(parsed.success, `${operation}: ${parsed.error?.message}`).toBe(true); + expect(capabilities.operations[operation]?.available, operation).toBe(true); + } catch (error) { + if (!(error instanceof UnsupportedMediaOperationError)) throw error; + expect(error.operation).toBe(operation); + expect(capabilities.operations[operation]?.available, operation).toBe(false); + } + } + }); + + it("rejects paths outside the media root", async () => { + const outside = await mkdtemp(path.join(tmpdir(), "media-outside-")); + try { + const stray = path.join(outside, "stray.mp4"); + await writeFile(stray, "not inside the root"); + await expect(provider.detectScenes({ path: stray })).rejects.toThrow(); + await expect( + provider.detectScenes({ + path: path.join(root, "..", path.basename(outside), "stray.mp4"), + }), + ).rejects.toThrow(); + await expect( + provider.redact({ + path: clip, + redactions: [], + outPath: path.join(outside, "escaped.mp4"), + }), + ).rejects.toThrow(); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it("redacts only inside the box", async () => { + const box = { x: 0.25, y: 0.25, w: 0.25, h: 0.25 }; + const pixels = { x: 80, y: 44, w: 80, h: 46 }; + for (const mode of ["solid", "blur"] as const) { + const outPath = path.join(root, `redacted-${mode}.mp4`); + const result = await provider.redact({ + path: clip, + outPath, + redactions: [ + { + keyframes: [ + { tMs: 0, box }, + { tMs: 2000, box }, + ], + mode, + paddingPct: 0, + featherPct: 0, + color: "#102030", + }, + ], + }); + expect(result.framesProcessed).toBeGreaterThan(0); + const source = await extractRgbFrame(clip, 1000); + const redacted = await extractRgbFrame(result.outPath, 1000); + expect(redacted.length).toBe(WIDTH * HEIGHT * 3); + const diff = regionDiff(source, redacted, pixels); + expect(diff.inside, mode).toBeGreaterThan(8); + expect(diff.outside, mode).toBeLessThan(1.5); + } + }); + + it("honours AbortSignal", async () => { + const aborted = new AbortController(); + aborted.abort(); + await expect( + provider.detectScenes({ path: clip, signal: aborted.signal }), + ).rejects.toMatchObject({ name: "AbortError" }); + await expect(provider.capabilities(aborted.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + + const long = await renderTestClip(path.join(root, "long.mp4"), { + seconds: 20, + width: 1280, + height: 720, + }); + const controller = new AbortController(); + const pending = provider.redact({ + path: long, + outPath: path.join(root, "aborted.mp4"), + redactions: [ + { + keyframes: [ + { tMs: 0, box: { x: 0.1, y: 0.1, w: 0.3, h: 0.3 } }, + { tMs: 20_000, box: { x: 0.5, y: 0.5, w: 0.3, h: 0.3 } }, + ], + mode: "blur", + paddingPct: 10, + featherPct: 0, + }, + ], + signal: controller.signal, + }); + setTimeout(() => controller.abort(), 150); + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + }); + }); +} diff --git a/packages/adapters/src/media-context.test.ts b/packages/adapters/src/media-context.test.ts new file mode 100644 index 0000000..37eec33 --- /dev/null +++ b/packages/adapters/src/media-context.test.ts @@ -0,0 +1,82 @@ +import type { AgentRuntime, AgentRuntimeEvent } from "@rakazo/adapter-kit"; +import type { PrismaClient } from "@rakazo/db"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; +import { createMediaModelClient, extractJson } from "./media-context.js"; + +function runtime(reply: string, compaction = true): AgentRuntime { + return { + describe: () => ({ capabilities: { compaction } }), + run: vi.fn(async function* (): AsyncGenerator { + yield { type: "done", text: reply }; + }), + } as unknown as AgentRuntime; +} + +function prisma(consented: boolean): PrismaClient { + return { + aiDataConsent: { findUnique: vi.fn(async () => (consented ? { userId: "u" } : null)) }, + } as unknown as PrismaClient; +} + +const model = async () => ({ provider: "openai", id: "gpt-test" }); +const input = { + spaceId: "s", + userId: "u", + instructions: "Reply with JSON.", + prompt: "Label", + schema: z.object({ title: z.string() }), +}; + +describe("extractJson", () => { + it("reads fenced, wrapped, and bare JSON", () => { + expect(extractJson('```json\n{"a":1}\n```')).toEqual({ a: 1 }); + expect(extractJson('Thinking... {"a":[1,2]} done')).toEqual({ a: [1, 2] }); + expect(extractJson("[1]")).toEqual([1]); + expect(extractJson("no json")).toBeUndefined(); + expect(extractJson("{broken")).toBeUndefined(); + }); +}); + +describe("createMediaModelClient", () => { + it("returns parsed output only with consent for the resolved recipient", async () => { + const withConsent = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"Beach"}'), + resolveModel: model, + }); + expect(await withConsent.generate(input)).toEqual({ title: "Beach" }); + + const run = runtime('{"title":"Beach"}'); + const withoutConsent = createMediaModelClient({ + prisma: prisma(false), + runtime: run, + resolveModel: model, + }); + expect(await withoutConsent.generate(input)).toBeNull(); + expect(run.run).not.toHaveBeenCalled(); + }); + + it("falls back to null for scripted runtimes, missing models, and invalid replies", async () => { + const scripted = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"x"}', false), + resolveModel: model, + }); + expect(await scripted.generate(input)).toBeNull(); + const noModel = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"x"}'), + resolveModel: async () => { + throw new Error("no model"); + }, + }); + expect(await noModel.generate(input)).toBeNull(); + const invalid = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"name":"x"}'), + resolveModel: model, + }); + expect(await invalid.generate(input)).toBeNull(); + }); +}); diff --git a/packages/adapters/src/media-context.ts b/packages/adapters/src/media-context.ts new file mode 100644 index 0000000..362a854 --- /dev/null +++ b/packages/adapters/src/media-context.ts @@ -0,0 +1,355 @@ +import type { + AgentInputImage, + AgentRunModel, + AgentRuntime, + ArtifactStore, + JobPublisher, + LinkShortener, + MediaInferenceProvider, + MediaPublisher, + MediaRenderer, + MediaSourceProvider, + MotionGraphicsRenderer, +} from "@rakazo/adapter-kit"; +import type { + AiAlteration, + ClientMediaItem, + CreateMarkInput, + Edl, + EdlError, + FfprobeReport, + GateReport, + IntroVariant, + MarketingCopy, + MarkTreatment, + MediaEditPlan, + MediaMark, + MediaPublication, + MediaRender, + MediaRenderOutput, + MediaRenderQuality, + ModelTier, + PlatformProfile, + Utm, +} from "@rakazo/contracts"; +import type { MediaRepos, PrismaClient } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import type { CompileSource, ProfileSet } from "@rakazo/media"; +import type { z } from "zod"; +import { aiRecipient } from "./ai-consent.js"; + +/* + * Shared wiring for Media Studio services. Composition roots (API, worker) + * build one MediaServiceDeps; every media service, job handler, and agent tool + * receives it instead of reaching for globals. + */ + +/** Realtime notifications the web and mobile clients subscribe to per space. */ +export type MediaRealtimeEvent = + | { type: "media.source"; sourceId: string; status: string; error?: string | null } + | { type: "media.asset"; assetId: string; ingestState: string } + | { type: "media.collections"; kind: "session" | "theme" } + | { type: "media.people" } + | { type: "media.render"; renderId: string; projectId: string; status: string; progress: number } + | { type: "media.publication"; publicationId: string; status: string }; + +/** + * Structured model calls on behalf of a user in a space. Returns null instead of + * calling a model when no model is configured, the runtime is scripted, or the + * user has not consented to sharing data with the resolved model recipient. + * Callers must then fall back to deterministic behavior. + */ +export interface MediaModelClient { + generate(input: { + spaceId: string; + userId: string; + /** Short system instructions; the reply must be one JSON document. */ + instructions: string; + prompt: string; + images?: AgentInputImage[]; + schema: z.ZodType; + tier?: ModelTier; + signal?: AbortSignal; + }): Promise; +} + +export interface MediaServiceDeps { + prisma: PrismaClient; + repos: MediaRepos; + jobs: JobPublisher; + /** Null when no sidecar is configured; media then stays hidden in the UI. */ + inference: MediaInferenceProvider | null; + renderer: MediaRenderer; + motion: MotionGraphicsRenderer; + publisher: MediaPublisher; + shortener: LinkShortener | null; + sources: Partial>; + /** Absolute DATA_DIR/media. Every stored media path is relative to it. */ + mediaRoot: string; + /** Builtin plus private platform profiles, cached by the composition root. */ + profiles: () => Promise; + artifacts: ArtifactStore; + model: MediaModelClient; + publish: (spaceId: string, event: MediaRealtimeEvent) => Promise; + now: () => Date; +} + +/** Realtime topic for a space's media events. */ +export function mediaTopic(spaceId: string): string { + return `media:${spaceId}`; +} + +const MODEL_TIMEOUT_MS = 120_000; +const RUNTIME_FAILURE = /^(?:I hit a problem:|Unknown model )/i; + +/** Extract the first JSON object or array from a model reply (models may wrap or think aloud). */ +export function extractJson(text: string): unknown { + const trimmed = text.trim(); + const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(trimmed); + const body = fenced?.[1] ?? trimmed; + const start = body.search(/[[{]/); + if (start < 0) return undefined; + const open = body[start]; + const close = open === "{" ? "}" : "]"; + const end = body.lastIndexOf(close); + if (end <= start) return undefined; + try { + return JSON.parse(body.slice(start, end + 1)); + } catch { + return undefined; + } +} + +export function createMediaModelClient(deps: { + prisma: PrismaClient; + runtime: AgentRuntime; + resolveModel: (scope: { + userId: string; + spaceId: string; + tier?: ModelTier; + }) => Promise; +}): MediaModelClient { + return { + async generate(input) { + if (!deps.runtime.describe().capabilities.compaction) return null; + let model: AgentRunModel; + try { + model = await deps.resolveModel({ + userId: input.userId, + spaceId: input.spaceId, + tier: input.tier ?? "fast", + }); + } catch { + return null; + } + if (model.provider === "scripted") return null; + const recipient = aiRecipient({ + provider: model.provider, + use: "model", + modelId: model.id, + baseUrl: model.baseUrl, + }); + if (!recipient) return null; + const consent = await deps.prisma.aiDataConsent.findUnique({ + where: { + userId_spaceId_recipientKey: { + userId: input.userId, + spaceId: input.spaceId, + recipientKey: recipient.key, + }, + }, + }); + if (!consent) return null; + const images = + input.images && model.acceptsImages !== false + ? input.images.slice(0, model.maxImagesPerPrompt ?? input.images.length) + : undefined; + const runId = `media-${crypto.randomUUID()}`; + const signal = input.signal + ? AbortSignal.any([input.signal, AbortSignal.timeout(MODEL_TIMEOUT_MS)]) + : AbortSignal.timeout(MODEL_TIMEOUT_MS); + let reply = ""; + try { + for await (const event of deps.runtime.run( + { + botId: "media-studio", + threadId: runId, + runId, + prompt: input.prompt, + instructions: input.instructions, + history: [], + tools: [], + model, + ...(images && images.length > 0 ? { currentTurnImages: images } : {}), + }, + { + operationId: runId, + traceId: runId, + spaceId: input.spaceId, + userId: input.userId, + signal, + }, + )) { + if (event.type === "done" && event.text) reply = event.text.trim(); + } + } catch (error) { + getLogger().warn("media model call failed", { + reason: error instanceof Error ? error.message : String(error), + }); + return null; + } + if (!reply || RUNTIME_FAILURE.test(reply)) return null; + const parsed = input.schema.safeParse(extractJson(reply)); + return parsed.success ? parsed.data : null; + }, + }; +} + +// --------------------------------------------------------------------------- +// Service contracts between media slices. Each slice implements its part; the +// API, worker, and agent tools only depend on these interfaces. +// --------------------------------------------------------------------------- + +export type CreateMarkInputType = z.infer; + +export interface MediaActor { + spaceId: string; + userId: string; +} + +export interface MediaIngestService { + /** Client-pushed metadata (Apple Photos, iOS): upsert, apply deletions, enqueue work. */ + pushItems( + actor: MediaActor, + input: { + sourceId: string; + items: ClientMediaItem[]; + deletedExternalIds: string[]; + syncToken: string | null; + complete: boolean; + }, + ): Promise<{ created: number; updated: number; unchanged: number; deleted: number }>; + /** Enqueue an incremental sync of a server-side source. */ + requestSync(actor: MediaActor, sourceId: string): Promise; + /** An upload finished: move the file into place and enqueue proxy and analysis. */ + uploadCompleted( + actor: MediaActor, + input: { assetId: string; tempPath: string; sha256: string; fileName: string }, + ): Promise; + recluster(actor: MediaActor): Promise; + /** Remove a source with its assets and every derived file. */ + removeSource(actor: MediaActor, sourceId: string): Promise; +} + +export interface MediaPreparedSources { + sources: Map; + /** Alterations applied while preparing (inpaint, synthetic identity, voice). */ + aiAlterations: AiAlteration[]; + /** Redactions that could not be honored and fell back, for the plan rationale. */ + fallbacks: string[]; +} + +export interface MediaPrivacyService { + /** + * Resolve every clip's source file and apply all face, mark, and region + * redactions from the EDL plus person and mark defaults. Fails closed: a + * redaction that cannot be applied throws instead of returning an unredacted source. + */ + prepareSources(input: { + actor: MediaActor; + projectId: string; + edl: Edl; + quality: MediaRenderQuality; + workDir: string; + signal?: AbortSignal; + }): Promise; + /** Re-detect on the rendered output and evaluate the full gate. */ + gate(input: { + actor: MediaActor; + projectId: string; + edl: Edl; + profile: PlatformProfile; + outputPath: string; + ffprobe: FfprobeReport; + copy: { title: string | null; caption: string | null; tags: string[] }; + bundleHasDisclosure: boolean; + autoFixRound: number; + signal?: AbortSignal; + }): Promise; + /** Create a mark: segment from the prompt, propagate, score, and suggest a fallback. */ + createMark(actor: MediaActor, input: CreateMarkInputType): Promise; + updateMark( + actor: MediaActor, + input: { + markId: string; + treatment?: MarkTreatment; + label?: string | null; + personId?: string | null; + }, + ): Promise; + /** Delete a person, their biometric data, and their files. */ + removePerson(actor: MediaActor, personId: string): Promise; +} + +export interface MediaRenderService { + start( + actor: MediaActor, + input: { planId: string; quality: MediaRenderQuality }, + ): Promise; + cancel(actor: MediaActor, renderId: string): Promise; + /** Absolute path of a finished output for streaming, or null. */ + outputPath(actor: MediaActor, renderId: string, fileName: string): Promise; +} + +export interface MediaMarketingService { + autoPlan( + actor: MediaActor, + input: { projectId: string; profileIds?: string[] }, + ): Promise; + submitPlan( + actor: MediaActor, + input: { + projectId: string; + edl: unknown; + rationale: string | null; + createdBy: "agent" | "user"; + }, + ): Promise<{ ok: true; plan: MediaEditPlan } | { ok: false; errors: EdlError[] }>; + introVariants( + actor: MediaActor, + input: { brandKitId: string; count: number }, + ): Promise; + thumbnails( + actor: MediaActor, + input: { renderId: string; count: number }, + ): Promise; + copy(actor: MediaActor, input: { renderId: string; language: string }): Promise; + createPublication( + actor: MediaActor, + input: { + renderId: string; + destination: string; + targetUrl: string | null; + utm: Utm | null; + title: string | null; + caption: string | null; + tags: string[]; + thumbnailVariant: string | null; + scheduledAt: string | null; + }, + ): Promise; + bundle(actor: MediaActor, publicationId: string): Promise; + cancelPublication(actor: MediaActor, publicationId: string): Promise; + importMetricsCsv( + actor: MediaActor, + input: { mappingId: string; csv: string }, + ): Promise<{ imported: number; skipped: Array<{ row: number; reason: string }> }>; +} + +export interface MediaServices { + deps: MediaServiceDeps; + ingest: MediaIngestService; + privacy: MediaPrivacyService; + render: MediaRenderService; + marketing: MediaMarketingService; +} diff --git a/packages/adapters/src/media-emulator-clips.ts b/packages/adapters/src/media-emulator-clips.ts new file mode 100644 index 0000000..95a3cd7 --- /dev/null +++ b/packages/adapters/src/media-emulator-clips.ts @@ -0,0 +1,66 @@ +import { spawnSync } from "node:child_process"; +import { runMediaTool } from "./media-folder-source-probe.js"; + +/* + * Synthetic test clips (ffmpeg testsrc2 + sine) for media tests across the + * codebase. Name the output with media-emulator markers to steer emulator + * results, e.g. `clip__faces-1__.mp4`. + */ + +export function ffmpegAvailable(ffmpegPath = "ffmpeg"): boolean { + return spawnSync(ffmpegPath, ["-version"], { stdio: "ignore" }).status === 0; +} + +export interface TestClipOptions { + seconds?: number; + width?: number; + height?: number; + fps?: number; + audio?: boolean; + ffmpegPath?: string; + signal?: AbortSignal; +} + +export async function renderTestClip(outPath: string, options: TestClipOptions = {}) { + const { seconds = 2, width = 320, height = 180, fps = 25, audio = true } = options; + const args = [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + `testsrc2=s=${width}x${height}:r=${fps}:d=${seconds}`, + ]; + if (audio) + args.push("-f", "lavfi", "-i", `sine=f=440:sample_rate=48000:d=${seconds}`, "-shortest"); + args.push("-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"); + if (audio) args.push("-c:a", "aac"); + args.push(outPath); + await runMediaTool(options.ffmpegPath ?? "ffmpeg", args, { signal: options.signal }); + return outPath; +} + +/** Decode one frame at `tMs` as packed RGB24 bytes. */ +export async function extractRgbFrame( + file: string, + tMs: number, + options: { ffmpegPath?: string } = {}, +): Promise { + const { stdout } = await runMediaTool(options.ffmpegPath ?? "ffmpeg", [ + "-v", + "error", + "-i", + file, + "-ss", + (tMs / 1000).toFixed(3), + "-frames:v", + "1", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "-", + ]); + return stdout; +} diff --git a/packages/adapters/src/media-emulator-source.ts b/packages/adapters/src/media-emulator-source.ts new file mode 100644 index 0000000..4d30b7e --- /dev/null +++ b/packages/adapters/src/media-emulator-source.ts @@ -0,0 +1,208 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { + MediaSourceItem, + MediaSourceListRequest, + MediaSourceListResult, + MediaSourceProvider, +} from "@rakazo/adapter-kit"; +import { PathOutsideRootError, resolveInside } from "@rakazo/media/node"; +import { z } from "zod"; +import type { MediaToolPaths } from "./media-folder-source-probe.js"; +import { runMediaTool } from "./media-folder-source-probe.js"; + +/* + * Deterministic synthetic library for tests and the scripted runtime. Config + * `{ count, seed }` describes `count` assets split into three capture sessions + * on separate days in three cities (more than 2 km apart), alternating two + * visual themes (`__theme-a__` testsrc2, `__theme-b__` SMPTE bars). Every fifth + * asset is a photo. File names carry media-emulator markers: + * index % 10 == 0 __faces-1____person-a__ + * index % 10 == 1 __faces-1____person-b__ + * index == 2 __nsfw-explicit_nudity-0.9__ + * index % 4 == 3 __speech__ (videos only) + * index % 10 == 6 __cut-2000__ (videos only) + * `resolve` renders the clip on first use (4 s, 320x180, ffmpeg testsrc2 or + * smptehdbars plus a sine tone) and caches it under `fixturesDir//`. + */ + +const ConfigSchema = z.object({ + count: z.number().int().min(1).max(100_000), + seed: z.number().int().min(0).max(2_147_483_647), +}); + +const CLIP_MS = 4000; +const CLIP_SIZE = { width: 320, height: 180, fps: 25 }; +const SESSIONS = [ + { lat: 52.52, lon: 13.405 }, + { lat: 53.5511, lon: 9.9937 }, + { lat: 48.1374, lon: 11.5755 }, +] as const; + +export interface EmulatorAsset { + index: number; + fileName: string; + kind: "video" | "photo"; + theme: "a" | "b"; + session: number; + capturedAt: Date; + gpsLat: number; + gpsLon: number; +} + +/** The deterministic description of asset `index` in an emulator library. */ +export function emulatorAsset(seed: number, count: number, index: number): EmulatorAsset { + const kind = index % 5 === 4 ? "photo" : "video"; + const theme = index % 2 === 0 ? "a" : "b"; + const session = Math.min(SESSIONS.length - 1, Math.floor((index * SESSIONS.length) / count)); + const firstInSession = Math.ceil((session * count) / SESSIONS.length); + const day = Date.UTC(2026, 4, 2 + (seed % 28) + session * 2, 8 + session); + const capturedAt = new Date(day + (index - firstInSession) * 3 * 60_000); + const place = SESSIONS[session] ?? SESSIONS[0]; + const jitter = ((index * 7919 + seed) % 100) / 100_000; + const markers = [`__theme-${theme}__`]; + if (index % 10 === 0) markers.push("__faces-1____person-a__"); + if (index % 10 === 1) markers.push("__faces-1____person-b__"); + if (index === 2) markers.push("__nsfw-explicit_nudity-0.9__"); + if (kind === "video" && index % 4 === 3) markers.push("__speech__"); + if (kind === "video" && index % 10 === 6) markers.push("__cut-2000__"); + const ext = kind === "photo" ? "jpg" : "mp4"; + return { + index, + fileName: `emu-${seed}-${String(index).padStart(5, "0")}${markers.join("")}.${ext}`, + kind, + theme, + session, + capturedAt, + gpsLat: Math.round((place.lat + jitter) * 1e5) / 1e5, + gpsLon: Math.round((place.lon + jitter) * 1e5) / 1e5, + }; +} + +function iso6709(lat: number, lon: number): string { + const sign = (value: number, width: number) => + `${value < 0 ? "-" : "+"}${Math.abs(value).toFixed(4).padStart(width, "0")}`; + return `${sign(lat, 7)}${sign(lon, 8)}/`; +} + +export interface EmulatorMediaSourceOptions extends MediaToolPaths { + /** Cache for rendered clips; must lie inside the media root to be analysable. */ + fixturesDir: string; +} + +export class EmulatorMediaSourceProvider implements MediaSourceProvider { + readonly kind = "emulator" as const; + + constructor(private readonly options: EmulatorMediaSourceOptions) {} + + async list(request: MediaSourceListRequest): Promise { + request.signal?.throwIfAborted(); + const { count, seed } = ConfigSchema.parse(request.config); + const start = request.cursor ? decodeIndex(request.cursor) : 0; + const end = Math.min(count, start + Math.max(1, Math.floor(request.limit))); + const items: MediaSourceItem[] = []; + for (let index = start; index < end; index++) { + const asset = emulatorAsset(seed, count, index); + items.push({ + externalId: asset.fileName, + kind: asset.kind, + capturedAt: asset.capturedAt, + durationMs: asset.kind === "video" ? CLIP_MS : null, + width: CLIP_SIZE.width, + height: CLIP_SIZE.height, + fps: asset.kind === "video" ? CLIP_SIZE.fps : null, + gpsLat: asset.gpsLat, + gpsLon: asset.gpsLon, + deviceModel: "Emulator", + albumNames: [], + isFavorite: index % 9 === 0, + fileName: asset.fileName, + sizeBytes: null, + revision: `emulator-v1-${seed}`, + }); + } + return { + items, + nextCursor: end < count ? Buffer.from(String(end)).toString("base64url") : null, + }; + } + + async resolve(config: Record, externalId: string): Promise { + const { count, seed } = ConfigSchema.parse(config); + const match = /^emu-(\d+)-(\d{5})/.exec(externalId); + const index = match ? Number(match[2]) : -1; + if (!match || Number(match[1]) !== seed || index >= count) + throw new PathOutsideRootError(externalId); + const asset = emulatorAsset(seed, count, index); + if (asset.fileName !== externalId) throw new PathOutsideRootError(externalId); + await mkdir(path.join(this.options.fixturesDir, String(seed)), { recursive: true }); + const target = await resolveInside(this.options.fixturesDir, `${seed}/${asset.fileName}`); + if (await exists(target)) return target; + const partial = path.join( + path.dirname(target), + `.partial-${randomUUID()}.${asset.kind === "photo" ? "jpg" : "mp4"}`, + ); + try { + await runMediaTool(this.options.ffmpegPath ?? "ffmpeg", renderArgs(asset, partial)); + await rename(partial, target); + } finally { + await rm(partial, { force: true }); + } + return target; + } +} + +function renderArgs(asset: EmulatorAsset, out: string): string[] { + const { width, height, fps } = CLIP_SIZE; + const pattern = asset.theme === "a" ? "testsrc2" : "smptehdbars"; + const video = `${pattern}=s=${width}x${height}:r=${fps}:d=${CLIP_MS / 1000}`; + const metadata = [ + "-metadata", + `creation_time=${asset.capturedAt.toISOString()}`, + "-metadata", + `location=${iso6709(asset.gpsLat, asset.gpsLon)}`, + ]; + if (asset.kind === "photo") + return ["-v", "error", "-y", "-f", "lavfi", "-i", video, "-frames:v", "1", out]; + return [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + video, + "-f", + "lavfi", + "-i", + `sine=f=${220 + (asset.index % 12) * 40}:sample_rate=48000:d=${CLIP_MS / 1000}`, + "-shortest", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + ...metadata, + "-movflags", + "+faststart", + out, + ]; +} + +function decodeIndex(cursor: string): number { + const value = Number(Buffer.from(cursor, "base64url").toString("utf8")); + if (!Number.isInteger(value) || value < 0) throw new Error("invalid emulator cursor"); + return value; +} + +async function exists(file: string): Promise { + try { + return (await stat(file)).isFile(); + } catch { + return false; + } +} diff --git a/packages/adapters/src/media-emulator.test.ts b/packages/adapters/src/media-emulator.test.ts new file mode 100644 index 0000000..8343a20 --- /dev/null +++ b/packages/adapters/src/media-emulator.test.ts @@ -0,0 +1,335 @@ +import { copyFile, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import { cosineSimilarity, haversineKm } from "@rakazo/media"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + buildRedactionGraph, + EmulatorMediaInferenceProvider, + parseEmulatorMarkers, + redactionRegions, +} from "./media-emulator.js"; +import { ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js"; +import { EmulatorMediaSourceProvider, emulatorAsset } from "./media-emulator-source.js"; +import { probeMedia } from "./media-folder-source-probe.js"; + +const hasFfmpeg = ffmpegAvailable(); +let root: string; +let provider: EmulatorMediaInferenceProvider; + +beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "media-emulator-")); + provider = new EmulatorMediaInferenceProvider({ root }); +}); + +afterAll(async () => { + await rm(root, { recursive: true, force: true }); +}); + +/** A non-media file: the emulator falls back to a 10 s clip, so no ffmpeg is needed. */ +async function fixture(name: string, content = "synthetic bytes") { + const file = path.join(root, name); + await writeFile(file, content); + return file; +} + +describe("parseEmulatorMarkers", () => { + it("reads every documented marker", () => { + expect( + parseEmulatorMarkers( + "/x/clip__faces-2____person-a____noface-gap-1000-2500____nsfw-explicit_nudity-0.9____nsfw-violence-0.4____cut-1500____cut-3000____speech____theme-b____inpaintq-0.3__.mp4", + ), + ).toEqual({ + faces: 2, + person: "a", + noFaceGaps: [{ startMs: 1000, endMs: 2500 }], + nsfw: { explicit_nudity: 0.9, violence: 0.4 }, + cutsMs: [1500, 3000], + speech: true, + theme: "b", + inpaintQuality: 0.3, + }); + expect(parseEmulatorMarkers("__nsfw-unknown-0.5____nsfw-violence-7__.mp4").nsfw).toEqual({}); + }); +}); + +describe("redaction regions", () => { + it("holds the union of neighbouring keyframes and bounds the graph", () => { + const regions = redactionRegions( + { + keyframes: [ + { tMs: 1000, box: { x: 0.5, y: 0.5, w: 0.1, h: 0.1 } }, + { tMs: 0, box: { x: 0.1, y: 0.1, w: 0.1, h: 0.1 } }, + ], + mode: "blur", + paddingPct: 0, + featherPct: 0, + }, + 1000, + 1000, + ); + expect(regions).toEqual([ + { startMs: 0, endMs: 1000, box: { x: 100, y: 100, w: 500, h: 500 } }, + { startMs: 1000, endMs: 1100, box: { x: 500, y: 500, w: 100, h: 100 } }, + ]); + const keyframes = Array.from({ length: 5000 }, (_, i) => ({ + tMs: i * 40, + box: { x: (i % 50) / 100, y: 0.2, w: 0.1, h: 0.1 }, + })); + const many = redactionRegions( + { keyframes, mode: "pixelate", paddingPct: 25, featherPct: 0 }, + 1920, + 1080, + ); + expect(many.length).toBeLessThanOrEqual(400); + expect(many[0]?.startMs).toBe(0); + expect(many.at(-1)?.endMs).toBe(4999 * 40 + 100); + const graph = buildRedactionGraph( + [{ keyframes, mode: "solid", paddingPct: 0, featherPct: 0, color: "#ff0000" }], + 1920, + 1080, + ); + expect(graph.filter).toContain("color=0xff0000@1:t=fill"); + }); +}); + +describe("EmulatorMediaInferenceProvider", () => { + it("is quiet without markers", async () => { + const file = await fixture("plain.mp4"); + const scenes = await provider.detectScenes({ path: file }); + expect(scenes.scenes).toEqual([{ startMs: 0, endMs: 10_000 }]); + expect(scenes.shots).toHaveLength(1); + expect((await provider.trackFaces({ path: file, sampleFps: 2 })).tracks).toEqual([]); + const faces = await provider.detectFaces({ path: file, sampleFps: 1 }); + expect(faces.frames).toHaveLength(10); + expect(faces.frames.every((frame) => frame.faces.length === 0)).toBe(true); + const classified = await provider.classifyContent({ path: file, sampleFps: 2 }); + expect(classified.frames).toHaveLength(20); + for (const frame of classified.frames) + expect(Object.values(frame.scores).every((score) => score === 0.01)).toBe(true); + expect(await provider.transcribe({ path: file })).toEqual({ language: null, words: [] }); + }); + + it("forces faces, gaps, cuts, classifier scores and speech from the file name", async () => { + const file = await fixture( + "rich__faces-2____noface-gap-2000-3000____cut-4000____nsfw-explicit_nudity-0.9____speech__.mp4", + ); + const tracks = await provider.trackFaces({ path: file, sampleFps: 2 }); + expect(tracks.tracks.map((track) => track.trackId)).toEqual(["track-1", "track-2"]); + const times = tracks.tracks[0]?.keyframes.map((keyframe) => keyframe.tMs) ?? []; + expect(times).toContain(1500); + expect(times).not.toContain(2000); + expect(times).not.toContain(2500); + expect(times).toContain(3000); + const gapFrame = await provider.detectFaces({ + path: file, + sampleFps: 1, + timesMs: [2500, 3500], + }); + expect(gapFrame.frames.map((frame) => frame.faces.length)).toEqual([0, 2]); + + expect((await provider.detectScenes({ path: file })).scenes).toEqual([ + { startMs: 0, endMs: 4000 }, + { startMs: 4000, endMs: 10_000 }, + ]); + const classified = await provider.classifyContent({ + path: file, + sampleFps: 0.5, + extraTimesMs: [4000, 4001], + }); + expect(classified.frames.map((frame) => frame.tMs)).toEqual([0, 2000, 4000, 4001, 6000, 8000]); + for (const frame of classified.frames) { + expect(frame.scores.explicit_nudity).toBe(0.9); + expect(frame.scores.violence).toBe(0.01); + } + const english = await provider.transcribe({ path: file }); + const german = await provider.transcribe({ path: file, language: "de-DE" }); + expect(english.language).toBe("en"); + expect(english.words.map((word) => word.text).slice(0, 2)).toEqual(["welcome", "back"]); + expect(german.language).toBe("de"); + expect(german.words[0]).toEqual({ + startMs: 400, + endMs: 720, + text: "willkommen", + confidence: 0.95, + }); + }); + + it("is deterministic per file content and keeps boxes stable", async () => { + const file = await fixture("same__faces-1__.mp4"); + const copy = path.join(root, "copy__faces-1__.mp4"); + await copyFile(file, copy); + const a = await provider.detectFaces({ path: file, sampleFps: 4 }); + const b = await provider.detectFaces({ path: file, sampleFps: 4 }); + const c = await provider.detectFaces({ path: copy, sampleFps: 4 }); + expect(a).toEqual(b); + expect(a).toEqual(c); + const boxes = a.frames.map((frame) => frame.faces[0]?.box); + expect(Math.abs((boxes.at(-1)?.x ?? 0) - (boxes[0]?.x ?? 0))).toBeLessThan(0.05); + }); + + it("embeds people and themes consistently", async () => { + const personA1 = await fixture("one__faces-1____person-a__.mp4", "first"); + const personA2 = await fixture("two__faces-1____person-a__.mp4", "second"); + const personB = await fixture("three__faces-1____person-b__.mp4", "third"); + const embed = async (file: string) => + (await provider.trackFaces({ path: file, sampleFps: 1 })).tracks[0]?.embedding ?? []; + expect(cosineSimilarity(await embed(personA1), await embed(personA2))).toBeCloseTo(1, 5); + expect(cosineSimilarity(await embed(personA1), await embed(personB))).toBeLessThan(0.8); + + const themeA1 = await fixture("x__theme-a__.mp4", "x"); + const themeA2 = await fixture("y__theme-a__.mp4", "y"); + const themeB = await fixture("z__theme-b__.mp4", "z"); + const mean = async (file: string) => + (await provider.embedFrames({ path: file, timesMs: [0, 1000] })).vectors[0] ?? []; + expect(cosineSimilarity(await mean(themeA1), await mean(themeA2))).toBeGreaterThan(0.95); + expect(cosineSimilarity(await mean(themeA1), await mean(themeB))).toBeLessThan(0.85); + }); + + it("never synthesizes identities", async () => { + const file = await fixture("synth.mp4"); + const capabilities = await provider.capabilities(); + expect(capabilities.operations.synthesizeIdentity).toMatchObject({ + available: false, + license: null, + }); + expect(capabilities.operations.inpaintVideo).toMatchObject({ + available: true, + experimental: true, + }); + await expect( + provider.synthesizeIdentity({ + path: file, + track: [], + seed: 1, + outPath: path.join(root, "s.mp4"), + }), + ).rejects.toBeInstanceOf(UnsupportedMediaOperationError); + }); + + it("rejects paths outside its root", async () => { + await expect(provider.detectScenes({ path: "/etc/hostname" })).rejects.toThrow( + "outside the allowed root", + ); + await expect(provider.detectScenes({ path: "relative.mp4" })).rejects.toThrow(); + }); + + it.skipIf(!hasFfmpeg)("writes masks and inpaints with the reported quality", async () => { + const clip = await renderTestClip(path.join(root, "mark__inpaintq-0.4__.mp4"), { + seconds: 1, + audio: false, + }); + const masks = await provider.segmentPrompted({ + path: clip, + prompt: { tMs: 0, point: { x: 0.5, y: 0.5 } }, + propagate: true, + outDir: path.join(root, "marks/m1"), + }); + expect((await readdir(masks.maskDir)).sort()).toEqual(["0.png", "500.png"]); + expect(masks.keyframes[0]?.box).toEqual({ x: 0.44, y: 0.44, w: 0.12, h: 0.12 }); + const inpainted = await provider.inpaintVideo({ + path: clip, + maskDir: masks.maskDir, + outPath: path.join(root, "marks/m1.mp4"), + }); + expect(inpainted.quality.score).toBe(0.4); + expect((await probeMedia(inpainted.outPath)).durationMs).toBeGreaterThan(900); + }); + + it.skipIf(!hasFfmpeg)("renders very large redaction graphs through a script file", async () => { + const clip = await renderTestClip(path.join(root, "busy.mp4"), { seconds: 1, audio: false }); + const instruction = (offset: number) => ({ + keyframes: Array.from({ length: 2000 }, (_, i) => ({ + tMs: i, + box: { x: offset + (i % 40) / 400, y: 0.1, w: 0.1, h: 0.1 }, + })), + mode: "blur" as const, + paddingPct: 0, + featherPct: 0, + }); + const instructions = [instruction(0), instruction(0.3), instruction(0.6)]; + expect(buildRedactionGraph(instructions, 320, 180).filter.length).toBeGreaterThan(100_000); + const result = await provider.redact({ + path: clip, + redactions: instructions, + outPath: path.join(root, "busy-redacted.mp4"), + }); + expect(result.framesProcessed).toBe(25); + }); + + it.skipIf(!hasFfmpeg)("measures loudness with ebur128", async () => { + const clip = await renderTestClip(path.join(root, "tone.mp4"), { seconds: 2 }); + const features = await provider.audioFeatures({ path: clip }); + expect(features.integratedLufs).toBeLessThan(-10); + expect(features.integratedLufs).toBeGreaterThan(-40); + expect(features.energy.length).toBeGreaterThan(0); + expect(features.beatsMs.length).toBeGreaterThan(0); + const silent = await renderTestClip(path.join(root, "mute.mp4"), { seconds: 1, audio: false }); + expect(await provider.audioFeatures({ path: silent })).toMatchObject({ + integratedLufs: null, + silences: [{ startMs: 0, endMs: 1000 }], + }); + }); +}); + +describe("EmulatorMediaSourceProvider", () => { + const config = { count: 30, seed: 4 }; + + it("describes three sessions, two themes and the marker assets", async () => { + const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); + const first = await source.list({ config, cursor: null, limit: 12 }); + const second = await source.list({ config, cursor: first.nextCursor, limit: 100 }); + expect(second.nextCursor).toBeNull(); + const items = [...first.items, ...second.items]; + expect(items).toHaveLength(30); + expect(new Set(items.map((item) => item.externalId)).size).toBe(30); + expect(await source.list({ config, cursor: null, limit: 12 })).toEqual(first); + + const assets = items.map((_, index) => emulatorAsset(config.seed, config.count, index)); + expect(new Set(assets.map((asset) => asset.session))).toEqual(new Set([0, 1, 2])); + expect(new Set(assets.map((asset) => asset.theme))).toEqual(new Set(["a", "b"])); + const sessionStarts = [0, 1, 2].map( + (session) => assets.find((asset) => asset.session === session)!, + ); + expect( + haversineKm( + { lat: sessionStarts[0]!.gpsLat, lon: sessionStarts[0]!.gpsLon }, + { lat: sessionStarts[1]!.gpsLat, lon: sessionStarts[1]!.gpsLon }, + ), + ).toBeGreaterThan(2); + const names = items.map((item) => item.externalId).join(" "); + expect(names).toContain("__faces-1____person-a__"); + expect(names).toContain("__faces-1____person-b__"); + expect(names).toContain("__nsfw-explicit_nudity-0.9__"); + expect(names).toContain("__speech__"); + expect(items.some((item) => item.kind === "photo")).toBe(true); + }); + + it("rejects unknown external ids", async () => { + const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); + for (const bad of [ + "../etc/passwd", + "emu-4-00099__theme-b__.mp4", + "emu-5-00000__theme-a__.mp4", + "emu-4-00000.mp4", + ]) + await expect(source.resolve(config, bad), bad).rejects.toThrow(); + }); + + it.skipIf(!hasFfmpeg)("renders and caches clips on demand", async () => { + const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); + const asset = emulatorAsset(config.seed, config.count, 0); + const file = await source.resolve(config, asset.fileName); + expect(path.basename(file)).toBe(asset.fileName); + const probed = await probeMedia(file); + expect(probed).toMatchObject({ durationMs: 4000, width: 320, height: 180, hasAudio: true }); + expect(probed.capturedAt?.toISOString()).toBe(asset.capturedAt.toISOString()); + expect(await source.resolve(config, asset.fileName)).toBe(file); + const tracks = await provider.trackFaces({ path: file, sampleFps: 1 }); + expect(tracks.tracks).toHaveLength(1); + const photo = emulatorAsset(config.seed, config.count, 4); + expect(photo.kind).toBe("photo"); + expect((await probeMedia(await source.resolve(config, photo.fileName))).width).toBe(320); + }); +}); diff --git a/packages/adapters/src/media-emulator.ts b/packages/adapters/src/media-emulator.ts new file mode 100644 index 0000000..a773840 --- /dev/null +++ b/packages/adapters/src/media-emulator.ts @@ -0,0 +1,1003 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, open, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { crc32, deflateSync } from "node:zlib"; +import type { + AudioFeaturesResult, + ClassifyContentResult, + DetectFacesResult, + DetectScenesResult, + EmbedFramesResult, + FaceLandmarksResult, + FaceObservation, + InpaintVideoResult, + MediaInferenceProvider, + MediaTimeSpan, + RedactionInstruction, + SaliencyResult, + SegmentPromptedResult, + TrackFacesResult, + TranscribeResult, + TranscriptWord, +} from "@rakazo/adapter-kit"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { + BoxKeyframe, + ContentCategory, + MediaCapabilities, + MediaMarkPrompt, + NormalizedBox, + NormalizedPoint, +} from "@rakazo/contracts"; +import { CONTENT_CATEGORIES, MediaInferenceOperationSchema } from "@rakazo/contracts"; +import { assertInside } from "@rakazo/media/node"; +import type { MediaToolPaths } from "./media-folder-source-probe.js"; +import { MediaToolMissingError, probeMedia, runMediaTool } from "./media-folder-source-probe.js"; + +/* + * Deterministic, offline MediaInferenceProvider. No ML: every result is derived + * from the file itself, so the same file always yields the same output. + * + * Seed: SHA-256 over the first 64 KiB of the file plus its byte size. Duration + * and frame size come from ffprobe; a file ffprobe cannot read counts as a + * 10 s, 1280x720 clip. + * + * Fixture markers in the file name force results (they combine freely, e.g. + * `clip__faces-2____speech____cut-1500__.mp4`): + * __faces-N__ N tracked faces with stable, slowly drifting + * boxes (track i keeps its box across calls). + * __person-__ identity seed for the face embeddings, so + * the same person matches across files. + * Without it, track i embeds as "track i". + * __noface-gap--__ no detection for any face inside that window + * (track keyframes are missing there). + * __nsfw--__ classifyContent returns `score` for that + * ContentCategory on every sampled frame. + * __cut-__ a shot and scene cut at that time. + * __speech__ transcribe returns a fixed word sequence + * (German when the request language starts + * with "de", English otherwise) with timings. + * __theme-__ frame embeddings share a theme vector + * (cosine >= 0.95 within a theme). + * __inpaintq-__ inpaintVideo reports this quality score. + * Without markers: zero faces, every classifier score 0.01, one shot and one + * scene, no words, embeddings from the file seed. + * + * redact and inpaintVideo write real files with ffmpeg (argv only): drawbox + * fills for solid/cover/sticker and boxblur or pixelation for blur, pixelate + * and partial modes, over keyframe boxes held between keyframes as their union + * (fail closed) and gated with enable='between(t,a,b)'. featherPct and + * landmarks are ignored; partial modes use fixed bands of the box. Output is lossless + * H.264, so pixels outside the boxes match the source. Without ffmpeg both + * throw MediaToolMissingError. synthesizeIdentity is always unsupported. + * + * audioFeatures measures loudness with ffmpeg ebur128 when available; tempo + * and beats are seeded numbers. Every path must lie inside the configured root. + */ + +const HEAD_BYTES = 64 * 1024; +const FALLBACK_DURATION_MS = 10_000; +const FALLBACK_SIZE = { width: 1280, height: 720 }; +const MAX_SAMPLES = 20_000; +/** How long the last keyframe box is held so the final frames stay covered. */ +const HOLD_AFTER_LAST_MS = 100; +const MAX_REGIONS_PER_INSTRUCTION = 400; +const INLINE_GRAPH_LIMIT = 100_000; +const EMBED_DIM = 32; +const IDENTITY_DIM = 16; +const CLASSIFIER_FLOOR = 0.01; + +const WORDS = { + en: "welcome back today we are filming at the lake and the light is perfect".split(" "), + de: "willkommen zurück heute drehen wir am see und das licht ist perfekt".split(" "), +}; + +export interface EmulatorMarkers { + faces: number; + person: string | null; + noFaceGaps: MediaTimeSpan[]; + nsfw: Partial>; + cutsMs: number[]; + speech: boolean; + theme: string | null; + inpaintQuality: number | null; +} + +/** Parse the fixture markers documented above from a file name. */ +export function parseEmulatorMarkers(fileName: string): EmulatorMarkers { + const name = path.basename(fileName); + const markers: EmulatorMarkers = { + faces: 0, + person: null, + noFaceGaps: [], + nsfw: {}, + cutsMs: [], + speech: name.includes("__speech__"), + theme: null, + inpaintQuality: null, + }; + for (const match of name.matchAll(/__faces-(\d{1,2})__/g)) markers.faces = Number(match[1]); + for (const match of name.matchAll(/__person-([a-z0-9]+)__/g)) markers.person = match[1] ?? null; + for (const match of name.matchAll(/__theme-([a-z0-9]+)__/g)) markers.theme = match[1] ?? null; + for (const match of name.matchAll(/__cut-(\d+)__/g)) markers.cutsMs.push(Number(match[1])); + for (const match of name.matchAll(/__noface-gap-(\d+)-(\d+)__/g)) { + const startMs = Number(match[1]); + const endMs = Number(match[2]); + if (endMs > startMs) markers.noFaceGaps.push({ startMs, endMs }); + } + for (const match of name.matchAll(/__nsfw-([a-z_]+)-(\d+(?:\.\d+)?)__/g)) { + const category = match[1] as ContentCategory; + const score = Number(match[2]); + if (CONTENT_CATEGORIES.includes(category) && score >= 0 && score <= 1) + markers.nsfw[category] = score; + } + for (const match of name.matchAll(/__inpaintq-(\d+(?:\.\d+)?)__/g)) { + const score = Number(match[1]); + if (score >= 0 && score <= 1) markers.inpaintQuality = score; + } + return markers; +} + +// --------------------------------------------------------------------------- +// Deterministic numbers +// --------------------------------------------------------------------------- + +/** mulberry32 seeded from SHA-256(seed + label). */ +function prng(seed: string, label: string): () => number { + let state = createHash("sha256").update(`${seed}\0${label}`).digest().readUInt32LE(0); + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function round4(value: number): number { + return Math.round(value * 10_000) / 10_000; +} + +function clampUnit(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +function unitVector(seed: string, label: string, dim: number): number[] { + const next = prng(seed, label); + const raw = Array.from({ length: dim }, () => next() * 2 - 1); + const norm = Math.hypot(...raw) || 1; + return raw.map((value) => round4(value / norm)); +} + +function sampleTimes(durationMs: number, sampleFps: number): number[] { + if (!(sampleFps > 0)) throw new RangeError("sampleFps must be positive"); + let step = Math.max(1, Math.round(1000 / sampleFps)); + while (durationMs / step > MAX_SAMPLES) step *= 2; + const times: number[] = []; + for (let t = 0; t < Math.max(durationMs, 1); t += step) times.push(t); + return times; +} + +// --------------------------------------------------------------------------- +// Provider +// --------------------------------------------------------------------------- + +interface Inspected { + absolute: string; + seed: string; + markers: EmulatorMarkers; + durationMs: number; + width: number; + height: number; + hasAudio: boolean; +} + +export interface EmulatorMediaInferenceOptions extends MediaToolPaths { + /** Media root; every input and output path must lie inside it. */ + root: string; +} + +export class EmulatorMediaInferenceProvider implements MediaInferenceProvider { + readonly id = "emulator"; + + constructor(private readonly options: EmulatorMediaInferenceOptions) {} + + async capabilities(signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const operations = {} as MediaCapabilities["operations"]; + for (const operation of MediaInferenceOperationSchema.options) { + operations[operation] = + operation === "synthesizeIdentity" + ? { + available: false, + experimental: true, + model: null, + license: null, + commercialUse: null, + } + : { + available: true, + experimental: operation === "inpaintVideo", + model: "emulator", + license: "Apache-2.0", + commercialUse: true, + }; + } + return { configured: true, operations, accelerator: "none", hardwareEncoders: [] }; + } + + async detectScenes(request: { path: string; signal?: AbortSignal }): Promise { + const file = await this.inspect(request.path, request.signal); + const cuts = [...new Set(file.markers.cutsMs)] + .filter((cut) => cut > 0 && cut < file.durationMs) + .sort((a, b) => a - b); + const bounds = [0, ...cuts, file.durationMs]; + const spans = bounds.slice(1).map((endMs, index) => ({ startMs: bounds[index] ?? 0, endMs })); + return { shots: spans.map((span) => ({ ...span, score: 1 })), scenes: spans }; + } + + async transcribe(request: { + path: string; + language?: string; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + if (!file.markers.speech) return { language: null, words: [] }; + const language = request.language?.toLowerCase().startsWith("de") ? "de" : "en"; + const words: TranscriptWord[] = []; + let cursor = 400; + for (const text of WORDS[language]) { + const endMs = cursor + 320; + if (endMs > file.durationMs) break; + words.push({ startMs: cursor, endMs, text, confidence: 0.95 }); + cursor = endMs + 80; + } + return { language, words }; + } + + async detectFaces(request: { + path: string; + sampleFps: number; + timesMs?: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const times = request.timesMs ?? sampleTimes(file.durationMs, request.sampleFps); + return { + frames: times.map((tMs) => ({ + tMs, + faces: this.facesAt(file, tMs).map(({ box, confidence }) => ({ + box, + confidence, + landmarks: landmarksFor(box), + })), + })), + }; + } + + async trackFaces(request: { + path: string; + sampleFps: number; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const times = sampleTimes(file.durationMs, request.sampleFps); + const tracks = Array.from({ length: file.markers.faces }, (_, index) => ({ + trackId: `track-${index + 1}`, + keyframes: times + .filter((tMs) => !inGap(file, tMs)) + .map((tMs) => ({ tMs, box: faceBox(file, index, tMs), confidence: 0.93 })), + embedding: unitVector( + "identity", + file.markers.person ? `${file.markers.person}:${index}` : `track:${index}`, + IDENTITY_DIM, + ), + })); + return { sampleFps: request.sampleFps, tracks }; + } + + async faceLandmarks(request: { + path: string; + timesMs: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + return { + frames: request.timesMs.map((tMs) => ({ + tMs, + faces: this.facesAt(file, tMs).map(({ box }) => ({ box, landmarks: landmarksFor(box) })), + })), + }; + } + + async embedFrames(request: { + path: string; + timesMs: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const base = unitVector( + "embed", + file.markers.theme ? `theme:${file.markers.theme}` : `asset:${file.seed}`, + EMBED_DIM, + ); + const vectors = request.timesMs.map((tMs) => { + const noise = prng(file.seed, `frame:${tMs}`); + const raw = base.map((value) => value + (noise() * 2 - 1) * 0.04); + const norm = Math.hypot(...raw) || 1; + return raw.map((value) => round4(value / norm)); + }); + return { model: "emulator-embed-v1", dim: EMBED_DIM, vectors }; + } + + async classifyContent(request: { + path: string; + sampleFps: number; + extraTimesMs?: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const times = [ + ...new Set([ + ...sampleTimes(file.durationMs, request.sampleFps), + ...(request.extraTimesMs ?? []), + ]), + ].sort((a, b) => a - b); + const scores = Object.fromEntries( + CONTENT_CATEGORIES.map((category) => [ + category, + file.markers.nsfw[category] ?? CLASSIFIER_FLOOR, + ]), + ) as Record; + return { + model: "emulator-classifier-v1", + frames: times.map((tMs) => ({ tMs, scores: { ...scores } })), + }; + } + + async segmentPrompted(request: { + path: string; + prompt: MediaMarkPrompt; + propagate: boolean; + outDir: string; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const outDir = await assertInside(this.options.root, request.outDir); + const box = promptBox(request.prompt); + const times = request.propagate + ? [...new Set([...sampleTimes(file.durationMs, 2), request.prompt.tMs])].sort((a, b) => a - b) + : [request.prompt.tMs]; + await mkdir(outDir, { recursive: true }); + const scale = Math.min(1, 640 / file.width); + const width = Math.max(2, Math.round(file.width * scale)); + const height = Math.max(2, Math.round(file.height * scale)); + const png = encodeMaskPng(width, height, box); + for (const tMs of times) { + request.signal?.throwIfAborted(); + await writeFile(path.join(outDir, `${tMs}.png`), png); + } + return { + maskDir: outDir, + keyframes: times.map((tMs) => ({ tMs, box, confidence: 0.9 })), + score: 0.9, + }; + } + + async inpaintVideo(request: { + path: string; + maskDir: string; + outPath: string; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const maskDir = await assertInside(this.options.root, request.maskDir); + const keyframes = await this.boxesFromMaskDir(maskDir, request.signal); + const outPath = await this.renderRedactions( + file, + [{ keyframes, mode: "blur", paddingPct: 0, featherPct: 0 }], + request.outPath, + request.signal, + ); + const score = file.markers.inpaintQuality ?? 0.82; + return { + outPath, + quality: { + temporalStability: file.markers.inpaintQuality ?? 0.85, + edgeQuality: file.markers.inpaintQuality ?? 0.8, + score, + }, + }; + } + + async redact(request: { + path: string; + redactions: RedactionInstruction[]; + outPath: string; + signal?: AbortSignal; + }): Promise<{ outPath: string; framesProcessed: number }> { + const file = await this.inspect(request.path, request.signal); + const instructions: RedactionInstruction[] = []; + for (const instruction of request.redactions) { + if (!instruction.maskDir) { + instructions.push(instruction); + continue; + } + const maskDir = await assertInside(this.options.root, instruction.maskDir); + const fromMasks = await this.boxesFromMaskDir(maskDir, request.signal); + instructions.push({ ...instruction, keyframes: [...instruction.keyframes, ...fromMasks] }); + } + const outPath = await this.renderRedactions( + file, + instructions, + request.outPath, + request.signal, + ); + const { stdout } = await runMediaTool( + this.ffprobe, + [ + "-v", + "error", + "-select_streams", + "v:0", + "-count_packets", + "-show_entries", + "stream=nb_read_packets", + "-of", + "csv=p=0", + "--", + outPath, + ], + { signal: request.signal }, + ); + const framesProcessed = Number.parseInt(stdout.toString("utf8").trim(), 10); + if (!Number.isFinite(framesProcessed) || framesProcessed <= 0) + throw new Error("redact produced no video frames"); + return { outPath, framesProcessed }; + } + + async synthesizeIdentity(request: { + path: string; + track: BoxKeyframe[]; + seed: number; + outPath: string; + signal?: AbortSignal; + }): Promise<{ outPath: string }> { + request.signal?.throwIfAborted(); + throw new UnsupportedMediaOperationError( + "synthesizeIdentity", + "no commercially licensed identity synthesis model is installed", + ); + } + + async audioFeatures(request: { + path: string; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const next = prng(file.seed, "audio"); + const tempoBpm = 90 + Math.floor(next() * 50); + const beatsMs: number[] = []; + const beatStep = 60_000 / tempoBpm; + for (let t = Math.round(next() * beatStep); t < file.durationMs; t += beatStep) + beatsMs.push(Math.round(t)); + const seeded = { + tempoBpm, + beatsMs, + energy: sampleTimes(file.durationMs, 2).map((tMs) => ({ + tMs, + value: round4(0.3 + next() * 0.5), + })), + silences: [] as MediaTimeSpan[], + }; + if (!file.hasAudio) + return { + integratedLufs: null, + truePeakDb: null, + tempoBpm: null, + beatsMs: [], + energy: [], + silences: [{ startMs: 0, endMs: file.durationMs }], + }; + let stderr: string; + try { + ({ stderr } = await runMediaTool( + this.ffmpeg, + [ + "-nostats", + "-hide_banner", + "-i", + file.absolute, + "-map", + "0:a:0", + "-af", + "ebur128=peak=true", + "-f", + "null", + "-", + ], + { signal: request.signal }, + )); + } catch (error) { + if (!(error instanceof MediaToolMissingError)) throw error; + return { + integratedLufs: round1(-24 + next() * 12), + truePeakDb: round1(-6 + next() * 5), + ...seeded, + }; + } + return { ...parseEbur128(stderr), tempoBpm, beatsMs }; + } + + async saliency(request: { + path: string; + timesMs: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + return { + points: request.timesMs.map((tMs) => { + const face = this.facesAt(file, tMs)[0]; + if (face) + return { + tMs, + cx: round4(face.box.x + face.box.w / 2), + cy: round4(face.box.y + face.box.h / 2), + strength: 0.9, + }; + const next = prng(file.seed, `saliency:${tMs}`); + return { + tMs, + cx: round4(0.4 + next() * 0.2), + cy: round4(0.4 + next() * 0.2), + strength: 0.5, + }; + }), + }; + } + + // ------------------------------------------------------------------------- + + private get ffmpeg(): string { + return this.options.ffmpegPath ?? "ffmpeg"; + } + + private get ffprobe(): string { + return this.options.ffprobePath ?? "ffprobe"; + } + + private async inspect(requested: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const absolute = await assertInside(this.options.root, requested); + const stats = await stat(absolute); + if (!stats.isFile()) throw new Error("media path is not a file"); + const handle = await open(absolute, "r"); + let head: Buffer; + try { + const buffer = Buffer.alloc(Math.min(HEAD_BYTES, stats.size)); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + head = buffer.subarray(0, bytesRead); + } finally { + await handle.close(); + } + const seed = createHash("sha256").update(head).update(String(stats.size)).digest("hex"); + let probed: Awaited> | null = null; + try { + probed = await probeMedia(absolute, { ffprobePath: this.ffprobe, signal }); + } catch { + signal?.throwIfAborted(); + } + return { + absolute, + seed, + markers: parseEmulatorMarkers(absolute), + durationMs: probed?.durationMs ?? FALLBACK_DURATION_MS, + width: probed?.width ?? FALLBACK_SIZE.width, + height: probed?.height ?? FALLBACK_SIZE.height, + hasAudio: probed?.hasAudio ?? false, + }; + } + + private facesAt(file: Inspected, tMs: number): FaceObservation[] { + if (inGap(file, tMs)) return []; + return Array.from({ length: file.markers.faces }, (_, index) => ({ + box: faceBox(file, index, tMs), + confidence: 0.93, + })); + } + + /** Bounding box per `.png` mask, measured at 128x128 and rounded outward. */ + private async boxesFromMaskDir(maskDir: string, signal?: AbortSignal): Promise { + const names = (await readdir(maskDir)).filter((name) => /^\d+\.png$/.test(name)); + const keyframes: BoxKeyframe[] = []; + for (const name of names.sort((a, b) => Number.parseInt(a, 10) - Number.parseInt(b, 10))) { + const { stdout } = await runMediaTool( + this.ffmpeg, + [ + "-v", + "error", + "-i", + path.join(maskDir, name), + "-vf", + "scale=128:128:flags=area", + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "-", + ], + { signal }, + ); + const box = maskBounds(stdout, 128, 128); + if (box) keyframes.push({ tMs: Number.parseInt(name, 10), box }); + } + return keyframes; + } + + private async renderRedactions( + file: Inspected, + instructions: RedactionInstruction[], + requestedOut: string, + signal?: AbortSignal, + ): Promise { + const outPath = await assertInside(this.options.root, requestedOut); + if (path.resolve(outPath) === path.resolve(file.absolute)) + throw new Error("redact output must differ from the input"); + await mkdir(path.dirname(outPath), { recursive: true }); + const graph = buildRedactionGraph(instructions, file.width, file.height); + // Linux caps one argv string at 128 KiB; larger graphs go through a script file. + const scratch = + graph.filter.length > INLINE_GRAPH_LIMIT + ? await mkdtemp(path.join(tmpdir(), "media-emulator-")) + : null; + try { + let filterArgs = ["-filter_complex", graph.filter]; + if (scratch) { + const script = path.join(scratch, "graph.txt"); + await writeFile(script, graph.filter); + filterArgs = ["-filter_complex_script", script]; + } + await runMediaTool( + this.ffmpeg, + [ + "-v", + "error", + "-y", + "-i", + file.absolute, + ...filterArgs, + "-map", + `[${graph.output}]`, + "-map", + "0:a?", + "-c:v", + "libx264", + "-qp", + "0", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-b:a", + "192k", + "-map_metadata", + "-1", + outPath, + ], + { signal }, + ); + } catch (error) { + await rm(outPath, { force: true }); + throw error; + } finally { + if (scratch) await rm(scratch, { recursive: true, force: true }); + } + return outPath; + } +} + +function round1(value: number): number { + return Math.round(value * 10) / 10; +} + +function inGap(file: Inspected, tMs: number): boolean { + return file.markers.noFaceGaps.some((gap) => tMs >= gap.startMs && tMs < gap.endMs); +} + +function faceBox(file: Inspected, index: number, tMs: number): NormalizedBox { + const next = prng(file.seed, `face:${index}`); + const column = index % 3; + const row = Math.floor(index / 3) % 2; + const drift = 0.04 * Math.min(1, tMs / Math.max(1, file.durationMs)); + return { + x: round4(0.06 + column * 0.3 + next() * 0.06 + drift), + y: round4(0.12 + row * 0.4 + next() * 0.1), + w: 0.16, + h: 0.22, + }; +} + +function landmarksFor(box: NormalizedBox): NormalizedPoint[] { + const at = (fx: number, fy: number) => ({ + x: round4(box.x + box.w * fx), + y: round4(box.y + box.h * fy), + }); + return [at(0.3, 0.38), at(0.7, 0.38), at(0.5, 0.58), at(0.35, 0.78), at(0.65, 0.78)]; +} + +function promptBox(prompt: MediaMarkPrompt): NormalizedBox { + if (prompt.box) return prompt.box; + if (!prompt.point) throw new Error("segment prompt needs a box or a point"); + const size = 0.12; + const x = clampUnit(Math.min(prompt.point.x - size / 2, 1 - size)); + const y = clampUnit(Math.min(prompt.point.y - size / 2, 1 - size)); + return { x: round4(x), y: round4(y), w: size, h: size }; +} + +function maskBounds(gray: Buffer, width: number, height: number): NormalizedBox | null { + let minX = width; + let minY = height; + let maxX = -1; + let maxY = -1; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + if ((gray[y * width + x] ?? 0) === 0) continue; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + if (maxX < 0) return null; + return { + x: minX / width, + y: minY / height, + w: (maxX + 1 - minX) / width, + h: (maxY + 1 - minY) / height, + }; +} + +/** Grayscale 8-bit PNG: white inside the box, black elsewhere. */ +export function encodeMaskPng(width: number, height: number, box: NormalizedBox): Buffer { + const x0 = Math.floor(box.x * width); + const y0 = Math.floor(box.y * height); + const x1 = Math.ceil((box.x + box.w) * width); + const y1 = Math.ceil((box.y + box.h) * height); + const raw = Buffer.alloc((width + 1) * height); + for (let y = 0; y < height; y++) { + const row = y * (width + 1); + if (y < y0 || y >= y1) continue; + raw.fill(255, row + 1 + Math.max(0, x0), row + 1 + Math.min(width, x1)); + } + const chunk = (type: string, data: Buffer) => { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "ascii"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(body) >>> 0); + return Buffer.concat([length, body, crc]); + }; + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; // bit depth + header[9] = 0; // grayscale + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", header), + chunk("IDAT", deflateSync(raw)), + chunk("IEND", Buffer.alloc(0)), + ]); +} + +// --------------------------------------------------------------------------- +// Redaction filtergraph +// --------------------------------------------------------------------------- + +interface PixelBox { + x: number; + y: number; + w: number; + h: number; +} + +interface Region { + startMs: number; + endMs: number; + box: PixelBox; +} + +function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox { + const x = Math.min(a.x, b.x); + const y = Math.min(a.y, b.y); + return { + x, + y, + w: Math.max(a.x + a.w, b.x + b.w) - x, + h: Math.max(a.y + a.h, b.y + b.h) - y, + }; +} + +function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox { + const padX = (box.w * Math.max(0, paddingPct)) / 200; + const padY = (box.h * Math.max(0, paddingPct)) / 200; + const x = clampUnit(box.x - padX); + const y = clampUnit(box.y - padY); + return { + x, + y, + w: clampUnit(box.x + box.w + padX) - x, + h: clampUnit(box.y + box.h + padY) - y, + }; +} + +/** Sub-box a partial mode covers, relative to the face box. */ +function modeBox(box: NormalizedBox, mode: RedactionInstruction["mode"]): NormalizedBox { + const band = (top: number, bottom: number) => ({ + x: box.x, + y: box.y + box.h * top, + w: box.w, + h: box.h * (bottom - top), + }); + if (mode === "partial_eyes") return band(0.22, 0.55); + if (mode === "partial_upper") return band(0, 0.55); + if (mode === "partial_lower") return band(0.5, 1); + return box; +} + +/** Normalized box → even pixel box (outward), at least 8x8, inside the frame. */ +function toPixels(box: NormalizedBox, width: number, height: number): PixelBox { + const even = (value: number) => value - (value % 2); + let x = even(Math.max(0, Math.floor(box.x * width))); + let y = even(Math.max(0, Math.floor(box.y * height))); + const right = Math.min(width, Math.ceil((box.x + box.w) * width)); + const bottom = Math.min(height, Math.ceil((box.y + box.h) * height)); + let w = Math.max(8, right - x + ((right - x) % 2)); + let h = Math.max(8, bottom - y + ((bottom - y) % 2)); + w = Math.min(w, even(width)); + h = Math.min(h, even(height)); + x = Math.min(x, even(width - w)); + y = Math.min(y, even(height - h)); + return { x, y, w, h }; +} + +/** Consecutive keyframes become one region holding the union of both boxes. */ +export function redactionRegions( + instruction: RedactionInstruction, + width: number, + height: number, +): Region[] { + const keyframes = [...instruction.keyframes] + .sort((a, b) => a.tMs - b.tMs) + .map((keyframe) => ({ + tMs: keyframe.tMs, + box: modeBox(padBox(keyframe.box, instruction.paddingPct), instruction.mode), + })); + const spans: Array<{ startMs: number; endMs: number; box: NormalizedBox }> = []; + keyframes.forEach((keyframe, index) => { + const following = keyframes[index + 1]; + spans.push( + following + ? { + startMs: keyframe.tMs, + endMs: following.tMs, + box: unionBox(keyframe.box, following.box), + } + : { startMs: keyframe.tMs, endMs: keyframe.tMs + HOLD_AFTER_LAST_MS, box: keyframe.box }, + ); + }); + // Bound graph size: merge neighbours (union, fail closed) until few enough remain. + let merged = spans; + while (merged.length > MAX_REGIONS_PER_INSTRUCTION) { + const next: typeof spans = []; + for (let index = 0; index < merged.length; index += 2) { + const a = merged[index]!; + const b = merged[index + 1]; + next.push(b ? { startMs: a.startMs, endMs: b.endMs, box: unionBox(a.box, b.box) } : a); + } + merged = next; + } + const regions: Region[] = []; + for (const span of merged) { + const box = toPixels(span.box, width, height); + const previous = regions.at(-1); + if ( + previous && + previous.endMs >= span.startMs && + previous.box.x === box.x && + previous.box.y === box.y && + previous.box.w === box.w && + previous.box.h === box.h + ) { + previous.endMs = Math.max(previous.endMs, span.endMs); + } else { + regions.push({ startMs: span.startMs, endMs: span.endMs, box }); + } + } + return regions; +} + +function drawColor(color: string | undefined): string { + return color && /^#[0-9a-fA-F]{6}$/.test(color) ? `0x${color.slice(1)}@1` : "black@1"; +} + +export function buildRedactionGraph( + instructions: RedactionInstruction[], + width: number, + height: number, +): { filter: string; output: string } { + const lines = ["[0:v:0]format=yuv420p[v0]"]; + let current = "v0"; + let counter = 0; + const label = () => `v${++counter}`; + for (const instruction of instructions) { + for (const region of redactionRegions(instruction, width, height)) { + const { x, y, w, h } = region.box; + const enable = `enable='between(t,${(region.startMs / 1000).toFixed(3)},${(region.endMs / 1000).toFixed(3)})'`; + const out = label(); + if ( + instruction.mode === "solid" || + instruction.mode === "cover" || + instruction.mode === "sticker" + ) { + lines.push( + `[${current}]drawbox=x=${x}:y=${y}:w=${w}:h=${h}:color=${drawColor(instruction.color)}:t=fill:${enable}[${out}]`, + ); + } else { + const base = label(); + const patch = label(); + const effected = label(); + let effect: string; + if (instruction.mode === "pixelate") { + const block = Math.max(4, Math.round(Math.min(w, h) / 8)); + effect = `scale=${Math.max(1, Math.ceil(w / block))}:${Math.max(1, Math.ceil(h / block))}:flags=area,scale=${w}:${h}:flags=neighbor`; + } else { + const radius = Math.max(1, Math.min(24, Math.floor(Math.min(w, h) / 4))); + effect = `boxblur=luma_radius=${radius}:luma_power=3:chroma_radius=${Math.max(1, Math.floor(radius / 2))}:chroma_power=3`; + } + lines.push(`[${current}]split=2[${base}][${patch}]`); + lines.push(`[${patch}]crop=${w}:${h}:${x}:${y},${effect}[${effected}]`); + lines.push(`[${base}][${effected}]overlay=${x}:${y}:${enable}[${out}]`); + } + current = out; + } + } + return { filter: lines.join(";\n"), output: current }; +} + +function parseEbur128( + stderr: string, +): Pick { + const summary = stderr.slice(stderr.lastIndexOf("Summary:")); + const integrated = /I:\s+(-?\d+(?:\.\d+)?) LUFS/.exec(summary); + const peak = /Peak:\s+(-?\d+(?:\.\d+)?) dBFS/.exec(summary); + const energy: Array<{ tMs: number; value: number }> = []; + const silences: MediaTimeSpan[] = []; + let silentSince: number | null = null; + let lastMs = 0; + for (const match of stderr.matchAll( + /t:\s*(\d+(?:\.\d+)?)\s+TARGET:\S+ LUFS\s+M:\s*(-?\d+(?:\.\d+)?)/g, + )) { + const tMs = Math.round(Number(match[1]) * 1000); + const momentary = Number(match[2]); + lastMs = tMs; + if (tMs % 500 < 100) energy.push({ tMs, value: round4(clampUnit((momentary + 70) / 70)) }); + if (momentary < -50) silentSince ??= tMs; + else if (silentSince !== null) { + if (tMs - silentSince >= 500) silences.push({ startMs: silentSince, endMs: tMs }); + silentSince = null; + } + } + if (silentSince !== null && lastMs - silentSince >= 500) + silences.push({ startMs: silentSince, endMs: lastMs }); + const integratedLufs = integrated ? Number(integrated[1]) : null; + return { + integratedLufs: integratedLufs !== null && integratedLufs > -70 ? integratedLufs : null, + truePeakDb: peak ? Number(peak[1]) : null, + energy, + silences, + }; +} diff --git a/packages/adapters/src/media-factory.test.ts b/packages/adapters/src/media-factory.test.ts new file mode 100644 index 0000000..f27c359 --- /dev/null +++ b/packages/adapters/src/media-factory.test.ts @@ -0,0 +1,53 @@ +import { homedir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + createMediaInferenceProvider, + createMediaSourceProvider, + mediaConfigFromEnv, + resolveMediaInferenceKind, +} from "./media-factory.js"; + +const worker = { + MEDIA_WORKER_URL: "http://127.0.0.1:8790", + MEDIA_WORKER_TOKEN: "placeholder-token-placeholder-token-00", +}; + +describe("media factory", () => { + it("reads one generic worker connection and derives paths", () => { + expect(mediaConfigFromEnv({ DATA_DIR: "/srv/evoo", ...worker })).toEqual({ + enabled: true, + workerUrl: worker.MEDIA_WORKER_URL, + workerToken: worker.MEDIA_WORKER_TOKEN, + profilesDir: path.join(homedir(), ".evoo/media-profiles"), + mediaRoot: "/srv/evoo/media", + }); + expect( + mediaConfigFromEnv({ DATA_DIR: "/srv/evoo", MEDIA_PROFILES_DIR: "/etc/evoo/profiles" }), + ).toMatchObject({ enabled: false, workerUrl: null, profilesDir: "/etc/evoo/profiles" }); + }); + + it("selects the sidecar only with URL and token", () => { + expect(resolveMediaInferenceKind(worker)).toBe("sidecar"); + expect(createMediaInferenceProvider(worker)?.id).toBe("sidecar"); + expect(resolveMediaInferenceKind({ MEDIA_WORKER_URL: worker.MEDIA_WORKER_URL })).toBeNull(); + expect(createMediaInferenceProvider({})).toBeNull(); + }); + + it("pins the emulator under the scripted runtime", () => { + expect(resolveMediaInferenceKind({ ...worker, AGENT_RUNTIME: "scripted" })).toBe("emulator"); + expect(createMediaInferenceProvider({ AGENT_RUNTIME: "scripted" })?.id).toBe("emulator"); + expect(mediaConfigFromEnv({ AGENT_RUNTIME: "scripted" }).enabled).toBe(true); + // pin-test-env sets AGENT_RUNTIME=scripted for every unit test. + expect(createMediaInferenceProvider()?.id).toBe("emulator"); + }); + + it("creates server-side sources only", () => { + expect(createMediaSourceProvider("folder", {}).kind).toBe("folder"); + expect(createMediaSourceProvider("emulator", { AGENT_RUNTIME: "scripted" }).kind).toBe( + "emulator", + ); + expect(() => createMediaSourceProvider("emulator", {})).toThrow(); + expect(() => createMediaSourceProvider("apple_photos", {})).toThrow(); + }); +}); diff --git a/packages/adapters/src/media-factory.ts b/packages/adapters/src/media-factory.ts new file mode 100644 index 0000000..e6453f1 --- /dev/null +++ b/packages/adapters/src/media-factory.ts @@ -0,0 +1,90 @@ +import { homedir } from "node:os"; +import path from "node:path"; +import type { MediaInferenceProvider, MediaSourceProvider } from "@rakazo/adapter-kit"; +import { expandHome } from "@rakazo/media/node"; +import { EmulatorMediaInferenceProvider } from "./media-emulator.js"; +import { EmulatorMediaSourceProvider } from "./media-emulator-source.js"; +import { FolderMediaSourceProvider } from "./media-folder-source.js"; +import { SidecarMediaInferenceProvider } from "./media-sidecar.js"; + +export interface MediaConfig { + /** Media Studio runs: a sidecar is configured or the scripted runtime uses the emulator. */ + enabled: boolean; + workerUrl: string | null; + workerToken: string | null; + profilesDir: string; + /** `${DATA_DIR}/media`, absolute. Every derived file and sidecar path lives here. */ + mediaRoot: string; +} + +export type MediaInferenceKind = "sidecar" | "emulator"; + +/** Emulators replace the sidecar whenever the scripted runtime is pinned (tests, e2e). */ +export function mediaEmulatorEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return env.AGENT_RUNTIME === "scripted"; +} + +export function mediaConfigFromEnv(env: NodeJS.ProcessEnv = process.env): MediaConfig { + const workerUrl = env.MEDIA_WORKER_URL?.trim() || null; + const workerToken = env.MEDIA_WORKER_TOKEN?.trim() || null; + return { + enabled: mediaEmulatorEnabled(env) || Boolean(workerUrl && workerToken), + workerUrl, + workerToken, + profilesDir: path.resolve( + expandHome(env.MEDIA_PROFILES_DIR?.trim() || "~/.evoo/media-profiles", homedir()), + ), + mediaRoot: path.resolve(env.DATA_DIR ?? "./data", "media"), + }; +} + +export function resolveMediaInferenceKind( + env: NodeJS.ProcessEnv = process.env, +): MediaInferenceKind | null { + if (mediaEmulatorEnabled(env)) return "emulator"; + const config = mediaConfigFromEnv(env); + return config.workerUrl && config.workerToken ? "sidecar" : null; +} + +/** The configured inference provider, or null when Media Studio is not set up. */ +export function createMediaInferenceProvider( + env: NodeJS.ProcessEnv = process.env, +): MediaInferenceProvider | null { + const config = mediaConfigFromEnv(env); + switch (resolveMediaInferenceKind(env)) { + case "emulator": + return new EmulatorMediaInferenceProvider({ root: config.mediaRoot }); + case "sidecar": + return new SidecarMediaInferenceProvider({ + baseUrl: config.workerUrl as string, + token: config.workerToken as string, + }); + default: + return null; + } +} + +/** Emulator clips are cached inside the media root so inference can read them. */ +export function emulatorFixturesDir(env: NodeJS.ProcessEnv = process.env): string { + return path.join(mediaConfigFromEnv(env).mediaRoot, "emulator-library"); +} + +/** + * Server-side source adapter for a persisted MediaSource kind. Client-pushed + * kinds (apple_photos, ios_library) have no server adapter. + */ +export function createMediaSourceProvider( + kind: string, + env: NodeJS.ProcessEnv = process.env, +): MediaSourceProvider { + switch (kind) { + case "folder": + return new FolderMediaSourceProvider(); + case "emulator": + if (!mediaEmulatorEnabled(env)) break; + return new EmulatorMediaSourceProvider({ fixturesDir: emulatorFixturesDir(env) }); + default: + break; + } + throw new Error(`No server-side media source for "${kind}". Use folder.`); +} diff --git a/packages/adapters/src/media-folder-source-probe.ts b/packages/adapters/src/media-folder-source-probe.ts new file mode 100644 index 0000000..c53a787 --- /dev/null +++ b/packages/adapters/src/media-folder-source-probe.ts @@ -0,0 +1,194 @@ +import { execFile } from "node:child_process"; + +/* + * Local ffmpeg/ffprobe helpers shared by the folder source and the media + * emulator. Every call is an argv array without a shell; nothing here builds a + * command string. + */ + +export class MediaToolMissingError extends Error { + constructor(readonly tool: string) { + super(`${tool} is not installed or not on PATH; install ffmpeg to use local media processing`); + this.name = "MediaToolMissingError"; + } +} + +export interface MediaToolPaths { + ffmpegPath?: string; + ffprobePath?: string; +} + +/** Run a media tool and collect stdout (binary) and stderr. Rejects on non-zero exit. */ +export function runMediaTool( + bin: string, + args: string[], + options: { signal?: AbortSignal; maxBuffer?: number } = {}, +): Promise<{ stdout: Buffer; stderr: string }> { + options.signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + execFile( + bin, + args, + { + encoding: "buffer", + maxBuffer: options.maxBuffer ?? 64 * 1024 * 1024, + signal: options.signal, + shell: false, + windowsHide: true, + }, + (error, stdout, stderr) => { + if (options.signal?.aborted) { + reject(options.signal.reason); + return; + } + if (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + reject(new MediaToolMissingError(bin)); + return; + } + const tail = stderr.toString("utf8").trim().split("\n").slice(-3).join(" "); + reject(new Error(`${bin} failed: ${tail || error.message}`)); + return; + } + resolve({ stdout, stderr: stderr.toString("utf8") }); + }, + ); + }); +} + +export interface ProbedMedia { + durationMs: number | null; + width: number | null; + height: number | null; + fps: number | null; + codec: string | null; + rotation: number; + hasAudio: boolean; + hasVideo: boolean; + capturedAt: Date | null; + gpsLat: number | null; + gpsLon: number | null; + deviceModel: string | null; +} + +interface FfprobeStream { + codec_type?: string; + codec_name?: string; + width?: number; + height?: number; + avg_frame_rate?: string; + r_frame_rate?: string; + duration?: string; + tags?: Record; + side_data_list?: Array<{ rotation?: number }>; +} + +interface FfprobeOutput { + streams?: FfprobeStream[]; + format?: { duration?: string; tags?: Record }; + frames?: Array<{ tags?: Record }>; +} + +/** Probe a local file. Throws when ffprobe is missing or cannot read the file. */ +export async function probeMedia( + file: string, + options: { ffprobePath?: string; signal?: AbortSignal; frameTags?: boolean } = {}, +): Promise { + const args = ["-v", "error", "-print_format", "json", "-show_format", "-show_streams"]; + if (options.frameTags) args.push("-show_frames", "-read_intervals", "%+#1"); + args.push("--", file); + const { stdout } = await runMediaTool(options.ffprobePath ?? "ffprobe", args, { + signal: options.signal, + }); + return parseFfprobe(JSON.parse(stdout.toString("utf8")) as FfprobeOutput); +} + +export function parseFfprobe(output: FfprobeOutput): ProbedMedia { + const streams = output.streams ?? []; + const video = streams.find((stream) => stream.codec_type === "video"); + const tags = collectTags(output); + const rotation = video ? streamRotation(video) : 0; + const quarterTurn = Math.abs(rotation) % 180 === 90; + const durationSeconds = Number(output.format?.duration ?? video?.duration); + const gps = parseIso6709( + tags.get("com.apple.quicktime.location.iso6709") ?? tags.get("location"), + ); + return { + durationMs: + Number.isFinite(durationSeconds) && durationSeconds > 0 + ? Math.round(durationSeconds * 1000) + : null, + width: (quarterTurn ? video?.height : video?.width) ?? null, + height: (quarterTurn ? video?.width : video?.height) ?? null, + fps: video ? (parseRate(video.avg_frame_rate) ?? parseRate(video.r_frame_rate)) : null, + codec: video?.codec_name ?? null, + rotation, + hasAudio: streams.some((stream) => stream.codec_type === "audio"), + hasVideo: Boolean(video), + capturedAt: parseCaptureDate( + tags.get("com.apple.quicktime.creationdate") ?? + tags.get("datetimeoriginal") ?? + tags.get("creation_time") ?? + tags.get("datetime"), + ), + gpsLat: gps?.lat ?? null, + gpsLon: gps?.lon ?? null, + deviceModel: tags.get("com.apple.quicktime.model") ?? tags.get("model") ?? null, + }; +} + +/** Format, stream and first-frame tags merged with lowercase keys; format wins. */ +function collectTags(output: FfprobeOutput): Map { + const merged = new Map(); + const sources = [ + output.format?.tags, + ...(output.streams ?? []).map((stream) => stream.tags), + ...(output.frames ?? []).map((frame) => frame.tags), + ]; + for (const source of sources) { + for (const [key, value] of Object.entries(source ?? {})) { + const normalized = key.toLowerCase(); + if (!merged.has(normalized) && typeof value === "string" && value.trim()) + merged.set(normalized, value.trim()); + } + } + return merged; +} + +function streamRotation(stream: FfprobeStream): number { + const side = stream.side_data_list?.find((entry) => typeof entry.rotation === "number"); + if (side?.rotation !== undefined) return side.rotation; + const tagged = Number(stream.tags?.rotate); + return Number.isFinite(tagged) ? tagged : 0; +} + +function parseRate(rate: string | undefined): number | null { + if (!rate) return null; + const [num, den] = rate.split("/").map(Number); + if (!num || !Number.isFinite(num)) return null; + const value = den ? num / den : num; + return Number.isFinite(value) && value > 0 ? Math.round(value * 1000) / 1000 : null; +} + +/** ISO 8601 or EXIF ("2026:05:01 09:00:00") timestamps; null when unparseable. */ +export function parseCaptureDate(value: string | undefined): Date | null { + if (!value) return null; + const exif = /^(\d{4}):(\d{2}):(\d{2})[ T](\d{2}):(\d{2}):(\d{2})/.exec(value); + const date = exif + ? new Date(`${exif[1]}-${exif[2]}-${exif[3]}T${exif[4]}:${exif[5]}:${exif[6]}Z`) + : new Date(value); + const time = date.getTime(); + return Number.isFinite(time) && time > 0 ? date : null; +} + +/** "+52.5200+013.4050+034.000/" → { lat, lon }; null when absent or out of range. */ +export function parseIso6709(value: string | undefined): { lat: number; lon: number } | null { + if (!value) return null; + const match = /^([+-]\d+(?:\.\d+)?)([+-]\d+(?:\.\d+)?)/.exec(value.trim()); + if (!match) return null; + const lat = Number(match[1]); + const lon = Number(match[2]); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return null; + if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null; + return { lat, lon }; +} diff --git a/packages/adapters/src/media-folder-source.test.ts b/packages/adapters/src/media-folder-source.test.ts new file mode 100644 index 0000000..dbb4ff6 --- /dev/null +++ b/packages/adapters/src/media-folder-source.test.ts @@ -0,0 +1,177 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { MediaSourceItem } from "@rakazo/adapter-kit"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { ffmpegAvailable } from "./media-emulator-clips.js"; +import { FolderMediaSourceProvider } from "./media-folder-source.js"; +import { parseIso6709, runMediaTool } from "./media-folder-source-probe.js"; + +let root: string; +const provider = new FolderMediaSourceProvider(); + +async function touch(relative: string, content = "") { + const file = path.join(root, relative); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, content); +} + +async function listAll(limit: number, cursor: string | null = null) { + const pages: MediaSourceItem[][] = []; + let next = cursor; + do { + const page = await provider.list({ config: { path: root }, cursor: next, limit }); + expect(page.items.length).toBeLessThanOrEqual(limit); + pages.push(page.items); + next = page.nextCursor; + } while (next); + return pages.flat().map((item) => item.externalId); +} + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "media-folder-")); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +describe("FolderMediaSourceProvider", () => { + it("lists media in stable code-unit order across pages", async () => { + for (const name of ["b/c.mp4", "b-d.jpg", "a.MOV", "b/a/z.webm", "c.png", "notes.txt"]) + await touch(name); + const ids = await listAll(2); + expect(ids).toEqual(["a.MOV", "b-d.jpg", "b/a/z.webm", "b/c.mp4", "c.png"]); + }); + + it("keeps the cursor stable when files are added mid-sync", async () => { + for (const name of ["a.mp4", "c.mp4", "e.mp4", "g.mp4"]) await touch(name); + const first = await provider.list({ config: { path: root }, cursor: null, limit: 2 }); + expect(first.items.map((item) => item.externalId)).toEqual(["a.mp4", "c.mp4"]); + await touch("b.mp4"); // before the cursor: picked up by the next full sync + await touch("d.mp4"); // after the cursor: part of this sync + const rest = await listAll(2, first.nextCursor); + expect(rest).toEqual(["d.mp4", "e.mp4", "g.mp4"]); + expect(await listAll(10)).toEqual(["a.mp4", "b.mp4", "c.mp4", "d.mp4", "e.mp4", "g.mp4"]); + }); + + it("skips hidden entries and symlinks without following them", async () => { + await touch("keep/clip.mp4"); + await touch(".hidden.mp4"); + await touch(".cache/inside.mp4"); + await touch("keep/.DS_Store.jpg"); + const outside = await mkdtemp(path.join(tmpdir(), "media-folder-outside-")); + try { + await writeFile(path.join(outside, "secret.mp4"), ""); + await symlink(path.join(outside, "secret.mp4"), path.join(root, "link.mp4")); + await symlink(outside, path.join(root, "linked-dir")); + await symlink(path.join(root, "keep"), path.join(root, "keep-alias")); + expect(await listAll(10)).toEqual(["keep/clip.mp4"]); + await expect(provider.resolve({ path: root }, "link.mp4")).rejects.toThrow(); + await expect(provider.resolve({ path: root }, "linked-dir/secret.mp4")).rejects.toThrow(); + await expect(provider.resolve({ path: root }, "keep-alias/clip.mp4")).rejects.toThrow(); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it("resolves inside the root and rejects traversal", async () => { + await touch("keep/clip.mp4"); + await touch(".hidden.mp4"); + await expect(provider.resolve({ path: root }, "keep/clip.mp4")).resolves.toBe( + path.join(root, "keep/clip.mp4"), + ); + for (const bad of [ + "../x", + "../x.mp4", + "keep/../../x.mp4", + "/etc/passwd", + ".hidden.mp4", + "keep", + ]) + await expect(provider.resolve({ path: root }, bad), bad).rejects.toThrow(); + }); + + it("rejects relative roots and forged cursors", async () => { + await expect( + provider.list({ config: { path: "media" }, cursor: null, limit: 5 }), + ).rejects.toThrow(); + await expect( + provider.list({ config: { path: root }, cursor: "not base64url!", limit: 5 }), + ).rejects.toThrow(); + }); + + it("pages a 1,000-file folder without duplicates or gaps", async () => { + const expected: string[] = []; + for (let dir = 0; dir < 10; dir++) { + for (let file = 0; file < 100; file++) { + const relative = `d${dir}/f${String(file).padStart(3, "0")}.${file % 3 === 0 ? "jpg" : "mp4"}`; + // Unreadable media must not abort a page; most files are empty and skip probing. + await touch(relative, file % 50 === 7 ? "garbage, not a video" : ""); + expected.push(relative); + } + } + await touch("d3/ignore.txt"); + const pages: MediaSourceItem[][] = []; + let cursor: string | null = null; + do { + const page = await provider.list({ config: { path: root }, cursor, limit: 137 }); + pages.push(page.items); + cursor = page.nextCursor; + } while (cursor); + const items = pages.flat(); + expect(pages.length).toBe(Math.ceil(1000 / 137)); + expect(items.map((item) => item.externalId)).toEqual(expected.sort()); + expect(new Set(items.map((item) => item.externalId)).size).toBe(1000); + const garbage = items.find((item) => item.externalId === "d0/f007.mp4"); + expect(garbage).toMatchObject({ durationMs: null, width: null, sizeBytes: 20, kind: "video" }); + expect(garbage?.capturedAt).toBeInstanceOf(Date); + expect(garbage?.revision).toMatch(/^\d+-20$/); + expect(items.find((item) => item.externalId === "d0/f000.jpg")?.kind).toBe("photo"); + }); + + it.skipIf(!ffmpegAvailable())( + "reads duration, size, capture time and GPS with ffprobe", + async () => { + await mkdir(path.join(root, "trip"), { recursive: true }); + await runMediaTool("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:r=25:d=1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-metadata", + "creation_time=2026-05-02T08:00:00Z", + "-metadata", + "location=+52.5200+013.4050/", + path.join(root, "trip/clip.mp4"), + ]); + const [item] = (await provider.list({ config: { path: root }, cursor: null, limit: 5 })) + .items; + expect(item).toMatchObject({ + externalId: "trip/clip.mp4", + kind: "video", + durationMs: 1000, + width: 320, + height: 180, + fps: 25, + gpsLat: 52.52, + gpsLon: 13.405, + albumNames: ["trip"], + }); + expect(item?.capturedAt?.toISOString()).toBe("2026-05-02T08:00:00.000Z"); + }, + ); + + it("parses ISO 6709 locations", () => { + expect(parseIso6709("+52.5200+013.4050+034.000/")).toEqual({ lat: 52.52, lon: 13.405 }); + expect(parseIso6709("-33.8688+151.2093/")).toEqual({ lat: -33.8688, lon: 151.2093 }); + expect(parseIso6709("+95.0+013.0/")).toBeNull(); + expect(parseIso6709("nowhere")).toBeNull(); + }); +}); diff --git a/packages/adapters/src/media-folder-source.ts b/packages/adapters/src/media-folder-source.ts new file mode 100644 index 0000000..7fa1d85 --- /dev/null +++ b/packages/adapters/src/media-folder-source.ts @@ -0,0 +1,221 @@ +import type { Dirent } from "node:fs"; +import { lstat, readdir } from "node:fs/promises"; +import path from "node:path"; +import type { + MediaSourceItem, + MediaSourceListRequest, + MediaSourceListResult, + MediaSourceProvider, +} from "@rakazo/adapter-kit"; +import type { MediaAssetKind } from "@rakazo/contracts"; +import { PathOutsideRootError, resolveInside } from "@rakazo/media/node"; +import { z } from "zod"; +import { probeMedia } from "./media-folder-source-probe.js"; + +/* + * Server-side folder source. The deployment owner picks an absolute root; the + * walk lists media files below it in a stable order (relative POSIX paths, + * code-unit sorted) so the opaque cursor (the last relative path) stays valid + * while files are added. Hidden entries and symlinks are skipped and never + * followed. Only the directories needed for one page are read. + */ + +export const FOLDER_MEDIA_EXTENSIONS: Readonly> = { + mp4: "video", + mov: "video", + m4v: "video", + mkv: "video", + webm: "video", + avi: "video", + jpg: "photo", + jpeg: "photo", + png: "photo", + heic: "photo", + heif: "photo", + webp: "photo", +}; + +const FolderConfigSchema = z.object({ + path: z.string().min(1).refine(path.isAbsolute, "folder path must be absolute"), +}); + +const MAX_PAGE = 1000; +const PROBE_CONCURRENCY = 4; + +export interface FolderMediaSourceOptions { + ffprobePath?: string; +} + +export function encodeFolderCursor(relativePath: string): string { + return Buffer.from(relativePath, "utf8").toString("base64url"); +} + +export function decodeFolderCursor(cursor: string | null): string | null { + if (!cursor) return null; + const decoded = Buffer.from(cursor, "base64url").toString("utf8"); + if (!decoded || decoded.includes("\0") || encodeFolderCursor(decoded) !== cursor) + throw new Error("invalid folder cursor"); + return decoded; +} + +function mediaKindOf(name: string): MediaAssetKind | null { + const ext = path.extname(name).slice(1).toLowerCase(); + return FOLDER_MEDIA_EXTENSIONS[ext] ?? null; +} + +/** Sort key that makes a depth-first walk yield relative paths in code-unit order. */ +function entryKey(entry: Dirent): string { + return entry.isDirectory() ? `${entry.name}/` : entry.name; +} + +function compareKeys(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** + * Yield relative paths of media files strictly after `after`, in order. + * Directories whose whole subtree sorts before the cursor are not read. + */ +async function* walkAfter( + root: string, + relativeDir: string, + after: string | null, + signal?: AbortSignal, +): AsyncGenerator { + signal?.throwIfAborted(); + let entries: Dirent[]; + try { + entries = await readdir(path.join(root, relativeDir), { withFileTypes: true }); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + // A directory removed or locked mid-walk is skipped; the root itself must exist. + if (relativeDir && (code === "ENOENT" || code === "EACCES" || code === "ENOTDIR")) return; + throw error; + } + const visible = entries + .filter((entry) => !entry.name.startsWith(".") && !entry.isSymbolicLink()) + .filter((entry) => entry.isDirectory() || (entry.isFile() && mediaKindOf(entry.name))) + .map((entry) => ({ entry, key: entryKey(entry) })) + .sort((a, b) => compareKeys(a.key, b.key)); + for (const { entry, key } of visible) { + const relative = relativeDir ? `${relativeDir}/${key}` : key; + if (entry.isDirectory()) { + // Subtree entirely before the cursor: every path in it starts with `relative`. + if (after !== null && relative < after && !after.startsWith(relative)) continue; + yield* walkAfter(root, relative.slice(0, -1), after, signal); + } else if (after === null || relative > after) { + yield relative; + } + } +} + +async function mapLimited( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const out = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const index = next++; + out[index] = await fn(items[index] as T); + } + }); + await Promise.all(workers); + return out; +} + +export class FolderMediaSourceProvider implements MediaSourceProvider { + readonly kind = "folder" as const; + + constructor(private readonly options: FolderMediaSourceOptions = {}) {} + + async list(request: MediaSourceListRequest): Promise { + const { path: root } = FolderConfigSchema.parse(request.config); + const limit = Math.max(1, Math.min(MAX_PAGE, Math.floor(request.limit))); + const after = decodeFolderCursor(request.cursor); + const relatives: string[] = []; + let more = false; + for await (const relative of walkAfter(root, "", after, request.signal)) { + if (relatives.length === limit) { + more = true; + break; + } + relatives.push(relative); + } + const items = await mapLimited(relatives, PROBE_CONCURRENCY, (relative) => + this.describe(root, relative, request.signal), + ); + const present = items.filter((item): item is MediaSourceItem => item !== null); + const last = relatives.at(-1); + return { items: present, nextCursor: more && last ? encodeFolderCursor(last) : null }; + } + + async resolve(config: Record, externalId: string): Promise { + const { path: root } = FolderConfigSchema.parse(config); + const segments = externalId.split("/"); + if (segments.some((segment) => segment.startsWith(".")) || !mediaKindOf(externalId)) + throw new PathOutsideRootError(externalId); + const absolute = await resolveInside(root, externalId); + // Symlinks are never followed, at any depth below the root. + let current = await resolveInside(root, "."); + for (const segment of segments) { + current = path.join(current, segment); + if ((await lstat(current)).isSymbolicLink()) throw new PathOutsideRootError(externalId); + } + if (!(await lstat(absolute)).isFile()) throw new PathOutsideRootError(externalId); + return absolute; + } + + /** + * Stat and probe one file. Probe failures yield null metadata, never a failed + * page. The parent folder name doubles as an album for rule-based labels. + */ + private async describe( + root: string, + relative: string, + signal?: AbortSignal, + ): Promise { + const absolute = path.join(root, relative); + let stats: Awaited>; + try { + stats = await lstat(absolute); + } catch { + return null; // Removed between walk and stat. + } + if (!stats.isFile()) return null; + const kind = mediaKindOf(relative) ?? "video"; + let probed: Awaited> | null = null; + if (stats.size > 0) { + try { + probed = await probeMedia(absolute, { + ffprobePath: this.options.ffprobePath, + signal, + frameTags: kind === "photo", + }); + } catch { + // Unreadable, unsupported, or no ffprobe on this host: keep the file with null metadata. + signal?.throwIfAborted(); + probed = null; + } + } + return { + externalId: relative, + kind, + capturedAt: probed?.capturedAt ?? new Date(Math.trunc(stats.mtimeMs)), + durationMs: kind === "video" ? (probed?.durationMs ?? null) : null, + width: probed?.width ?? null, + height: probed?.height ?? null, + fps: kind === "video" ? (probed?.fps ?? null) : null, + gpsLat: probed?.gpsLat ?? null, + gpsLon: probed?.gpsLon ?? null, + deviceModel: probed?.deviceModel ?? null, + albumNames: relative.includes("/") ? [path.posix.basename(path.posix.dirname(relative))] : [], + isFavorite: false, + fileName: path.posix.basename(relative), + sizeBytes: stats.size, + revision: `${Math.trunc(stats.mtimeMs)}-${stats.size}`, + }; + } +} diff --git a/packages/adapters/src/media-marketing-copy.ts b/packages/adapters/src/media-marketing-copy.ts new file mode 100644 index 0000000..f8ed067 --- /dev/null +++ b/packages/adapters/src/media-marketing-copy.ts @@ -0,0 +1,164 @@ +import type { Edl, MarketingCopy, MediaSegment, PlatformProfile } from "@rakazo/contracts"; +import { MarketingCopySchema } from "@rakazo/contracts"; +import { InvalidMediaInputError, MediaStateConflictError } from "@rakazo/db"; +import { checkCopy, edlDurationMs, isShortForm, normalizeHashtags } from "@rakazo/media"; +import { z } from "zod"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Platform copy for a render (spec 10.2). The model writes from the plan's + * metadata and the transcript of the clips actually used, under the + * profile's limits; it is told to state nothing the material does not show. + * Without a model (or consent) the copy comes from the plan metadata and the + * project title only. Either way hashtags are normalized and capped, long + * fields are cut at a word boundary to the profile limits, and the result is + * checked like the gate's text check; findings are returned as `issues`. + */ + +const TRANSCRIPT_CHARS = 1500; +const CHAPTER_MIN_PROGRAM_MS = 60_000; + +const CopyReplySchema = z.object({ + title: z.string().max(200), + hook: z.string().max(200), + caption: z.string().max(5000), + hashtags: z.array(z.string().max(60)).max(50), + description: z.string().max(5000).optional(), + chapters: z + .array(z.object({ atMs: z.number().int().min(0), title: z.string().max(100) })) + .max(100) + .optional(), + altText: z.string().max(500).optional(), +}); +type CopyReply = z.infer; + +const LANGUAGE = /^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})?$/; + +/** Cut to `max` characters, at the last word boundary when there is one. */ +export function truncateText(value: string, max: number): string { + const chars = [...value.trim()]; + if (chars.length <= max) return value.trim(); + const cut = chars.slice(0, Math.max(0, max)).join(""); + const space = cut.lastIndexOf(" "); + return (space > max * 0.6 ? cut.slice(0, space) : cut).trim(); +} + +function transcriptOf(edl: Edl, segments: MediaSegment[]): string { + const byId = new Map(segments.map((segment) => [segment.id, segment])); + const parts: string[] = []; + for (const clip of [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs)) { + const linked = clip.segmentId ? byId.get(clip.segmentId) : undefined; + const overlapping = linked + ? [linked] + : segments.filter( + (s) => s.assetId === clip.assetId && s.startMs < clip.outMs && s.endMs > clip.inMs, + ); + for (const segment of overlapping) { + const text = segment.transcriptExcerpt?.replace(/\s+/g, " ").trim(); + if (text && !parts.includes(text)) parts.push(text); + } + } + return parts.join(" ").slice(0, TRANSCRIPT_CHARS); +} + +function wantsChapters(profile: PlatformProfile, edl: Edl): boolean { + return !isShortForm(profile) && edlDurationMs(edl) >= CHAPTER_MIN_PROGRAM_MS; +} + +function fallbackCopy(edl: Edl, projectTitle: string, transcript: string): CopyReply { + const title = edl.metadata.title?.trim() || projectTitle; + const sentence = transcript.split(/(?<=[.!?])\s+/)[0]?.trim() ?? ""; + return { + title, + hook: sentence && sentence.length <= 120 ? sentence : title, + caption: edl.metadata.description?.trim() || title, + hashtags: edl.metadata.tags, + ...(edl.metadata.description ? { description: edl.metadata.description } : {}), + }; +} + +function instructions(profile: PlatformProfile, language: string, chapters: boolean): string { + const { text } = profile; + const link = + text.linkPolicy === "allowed" + ? "Links are allowed." + : text.linkPolicy === "bio_only" + ? 'Never put a link in the text; say "link in bio" if a call to action needs one.' + : "Never mention or include links."; + return [ + `You write social video copy in the language "${language}" for ${profile.name}.`, + "Use only facts from the given title, description, tags and transcript. Never invent claims, prices, results, quotes, guarantees or urgency.", + "No clickbait that misrepresents the video, no engagement bait, no promises the video does not keep.", + `Limits: title ${text.titleMaxChars ?? 100} characters, hook 120 characters, caption ${text.captionMaxChars} characters, at most ${text.hashtagLimit} hashtags without the # sign.`, + link, + chapters + ? "Add chapters (atMs from 0, at least 10 s apart) only where the transcript shows topic changes." + : "Do not add chapters.", + "altText describes what is visible for screen readers, factually.", + 'Reply with one JSON object: {"title","hook","caption","hashtags":[],"description","chapters":[{"atMs","title"}],"altText"}.', + ].join("\n"); +} + +export async function generateCopy( + deps: MediaServiceDeps, + actor: MediaActor, + input: { renderId: string; language: string }, +): Promise { + const language = input.language.trim(); + if (!LANGUAGE.test(language)) throw new InvalidMediaInputError("Invalid language code"); + const { spaceId } = actor; + const render = await deps.repos.getRender(spaceId, input.renderId); + const [plan, profileSet, project] = await Promise.all([ + deps.repos.getPlan(spaceId, render.planId), + deps.profiles(), + deps.repos.getProjectRecord(spaceId, render.projectId), + ]); + const profile = profileSet.profiles.get(render.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${render.profileId} is not available`); + const edl = plan.edl; + const segments = await deps.repos.listSegments(spaceId, [ + ...new Set(edl.tracks.video.map((clip) => clip.assetId)), + ]); + const transcript = transcriptOf(edl, segments); + const chapters = wantsChapters(profile, edl); + const durationMs = edlDurationMs(edl); + + const reply = await deps.model.generate({ + spaceId, + userId: actor.userId, + instructions: instructions(profile, language, chapters), + prompt: JSON.stringify({ + projectTitle: project.title, + title: edl.metadata.title ?? null, + description: edl.metadata.description ?? null, + tags: edl.metadata.tags, + durationMs, + transcript: transcript || null, + }), + schema: CopyReplySchema, + }); + const draft = reply ?? fallbackCopy(edl, project.title, transcript); + + const title = truncateText(draft.title, Math.min(200, profile.text.titleMaxChars ?? 200)); + const caption = truncateText(draft.caption, Math.min(5000, profile.text.captionMaxChars)); + const hashtags = normalizeHashtags(draft.hashtags, profile.text.hashtagLimit).map((tag) => + tag.slice(0, 60), + ); + const chapterList = chapters + ? (draft.chapters ?? []) + .filter((chapter) => chapter.atMs < durationMs && chapter.title.trim()) + .sort((a, b) => a.atMs - b.atMs) + : []; + return MarketingCopySchema.parse({ + profileId: profile.id, + language, + title, + hook: truncateText(draft.hook, 200), + caption, + hashtags, + ...(draft.description ? { description: truncateText(draft.description, 5000) } : {}), + ...(chapterList.length > 0 ? { chapters: chapterList } : {}), + ...(draft.altText?.trim() ? { altText: truncateText(draft.altText, 500) } : {}), + issues: checkCopy(profile, { title, caption, tags: hashtags }), + }); +} diff --git a/packages/adapters/src/media-marketing-files.ts b/packages/adapters/src/media-marketing-files.ts new file mode 100644 index 0000000..534c9c2 --- /dev/null +++ b/packages/adapters/src/media-marketing-files.ts @@ -0,0 +1,124 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { AdapterContext } from "@rakazo/adapter-kit"; +import type { MediaRenderOutput } from "@rakazo/contracts"; +import { MediaRenderOutputSchema } from "@rakazo/contracts"; +import { IsolationError } from "@rakazo/db"; +import { resolveInside } from "@rakazo/media/node"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * File and artifact plumbing shared by the marketing services. Media artifacts + * (thumbnails, intro posters, small bundles) are space artifacts without a bot, + * owned by the acting user, like chat attachments. + */ + +/** A stored render output: the public fields plus its path relative to the media root. */ +export type StoredRenderOutput = MediaRenderOutput & { path?: string }; + +export function mediaAdapterContext(actor: MediaActor, operationId: string): AdapterContext { + return { operationId, traceId: operationId, spaceId: actor.spaceId, userId: actor.userId }; +} + +export async function storeMediaArtifact( + deps: Pick, + actor: MediaActor, + input: { name: string; mimeType: string; bytes: Uint8Array }, +): Promise { + const context = mediaAdapterContext(actor, `media-artifact:${input.name}`); + const stored = await deps.artifacts.put(input, context); + const row = await deps.prisma.artifact + .create({ + data: { + spaceId: actor.spaceId, + userId: actor.userId, + name: input.name, + mimeType: input.mimeType, + size: input.bytes.byteLength, + hash: createHash("sha256").update(input.bytes).digest("hex"), + storageKey: stored.id, + }, + select: { id: true }, + }) + .catch(async (error) => { + await deps.artifacts.remove(stored.id, context).catch(() => undefined); + throw error; + }); + return row.id; +} + +/** + * Copy a space artifact (brand font, logo) into `dir` so a local renderer can + * read it. The file name is the artifact id plus its sanitized extension. + */ +export async function materializeArtifact( + deps: Pick, + actor: MediaActor, + artifactId: string, + dir: string, +): Promise { + const row = await deps.prisma.artifact.findFirst({ + where: { id: artifactId, spaceId: actor.spaceId }, + select: { id: true, name: true, storageKey: true }, + }); + if (!row) throw new IsolationError(); + const bytes = await deps.artifacts.get( + row.storageKey, + mediaAdapterContext(actor, `media-artifact-get:${row.id}`), + ); + const ext = path.extname(row.name).toLowerCase(); + const safeId = row.id.replace(/[^A-Za-z0-9_-]/g, ""); + const file = path.join(dir, `${safeId}${/^\.[a-z0-9]{1,5}$/.test(ext) ? ext : ""}`); + await mkdir(dir, { recursive: true }); + await writeFile(file, bytes); + return file; +} + +/** Outputs of a render row, dropping anything that does not parse. */ +export function storedRenderOutputs(outputs: unknown): StoredRenderOutput[] { + if (!Array.isArray(outputs)) return []; + return outputs.flatMap((raw) => { + const parsed = MediaRenderOutputSchema.safeParse(raw); + if (!parsed.success) return []; + const internal = (raw as { path?: unknown }).path; + return [{ ...parsed.data, ...(typeof internal === "string" ? { path: internal } : {}) }]; + }); +} + +export function publicRenderOutput(output: StoredRenderOutput): MediaRenderOutput { + return { + kind: output.kind, + artifactId: output.artifactId, + fileName: output.fileName, + sizeBytes: output.sizeBytes, + }; +} + +/** + * Absolute path of a render output: its stored relative path, or + * `/renders//` when the renderer stored none. + */ +export function renderOutputPath( + deps: Pick, + spaceId: string, + renderId: string, + output: StoredRenderOutput, +): Promise { + return resolveInside( + deps.mediaRoot, + output.path ?? path.join(spaceId, "renders", renderId, path.basename(output.fileName)), + ); +} + +/** Lowercase ASCII slug for UTM values and file names; empty when nothing is left. */ +export function slugify(value: string, max = 100): string { + return value + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, max) + .replace(/-+$/g, ""); +} diff --git a/packages/adapters/src/media-marketing-plan.ts b/packages/adapters/src/media-marketing-plan.ts new file mode 100644 index 0000000..ff9377e --- /dev/null +++ b/packages/adapters/src/media-marketing-plan.ts @@ -0,0 +1,532 @@ +import { mkdir, readFile, rm } from "node:fs/promises"; +import path from "node:path"; +import type { AgentInputImage } from "@rakazo/adapter-kit"; +import type { + BrandKit, + ContentCategory, + Edl, + EdlError, + MediaEditPlan, + MediaProjectGoal, + MediaSegment, + PlatformProfile, + TeaserWeights, +} from "@rakazo/contracts"; +import { TEASER_LEARNING_MIN_SAMPLES } from "@rakazo/contracts"; +import type { MediaProjectRecord, MediaRepos } from "@rakazo/db"; +import { InvalidMediaInputError } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import type { EdlContext, ScoredSegmentInput, SegmentFeatures } from "@rakazo/media"; +import { + autoPlanEdl, + defaultTeaserWeights, + introMs, + scoreSegment, + speechHookScore, + validateEdl, +} from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import { z } from "zod"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Edit plans for a project: the deterministic first cut per profile, an + * optional model ranking of the teaser candidates, and validation of plans the + * agent or user submits. + * + * Model judge (spec 8): only the top JUDGE_TOP_K candidates by deterministic + * score go to the model, and only those whose stored classifier scores already + * pass the target profile thresholds and that show no face of a protected + * person (or an unassigned face next to one). The model receives one still + * frame per candidate, taken from the proxy, plus the transcript excerpt. + * Face crops, embeddings and person data never leave the host. The model's + * order re-weights the candidates: rank r of n scales every feature by + * 1 - r / (2n); every usable segment it did not rank keeps half its weight + * (paid-only segments stay as they are: they only mark the payoff). Without a + * model, consent, or a usable reply the deterministic order stands. + */ + +export const JUDGE_TOP_K = 12; +const JUDGE_FRAME_WIDTH = 320; +const TRANSCRIPT_EXCERPT_CHARS = 300; +const RATIONALE_MAX_CHARS = 2000; + +const RANKING_GOALS: ReadonlySet = new Set([ + "teaser", + "trailer", + "highlight", + "story", +]); + +const JudgeReplySchema = z.object({ + ranking: z + .array(z.object({ segmentId: z.string().min(1).max(100), reason: z.string().max(300) })) + .max(50), +}); + +type AssetRow = Awaited>[number]; +type FaceTrackRow = Awaited>[number]; +type PersonRow = Awaited>[number]; +type ClassifierFrames = Array<{ tMs: number; scores: Partial> }>; + +export interface PlanContext { + project: MediaProjectRecord; + kit: BrandKit | null; + assets: Map; + segments: MediaSegment[]; + faceTracks: FaceTrackRow[]; + /** Every person of the space; plans may reference any of them. */ + people: PersonRow[]; + markIds: Set; + marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; + profiles: Map; +} + +export async function loadPlanContext( + deps: MediaServiceDeps, + actor: MediaActor, + projectId: string, +): Promise { + const { repos } = deps; + const { spaceId } = actor; + const project = await repos.getProjectRecord(spaceId, projectId); + const fromCollections = await repos.collectionAssetIds(spaceId, project.collectionIds); + const assetIds = [...new Set([...project.assetIds, ...fromCollections])]; + const [assetRows, segments, faceTracks, people, marks, kit, profileSet] = await Promise.all([ + repos.listAssetRecords(spaceId, assetIds), + repos.listSegments(spaceId, assetIds), + repos.faceTrackRecords(spaceId, assetIds), + repos.listPersonRecords(spaceId), + repos.listMarks(spaceId), + project.brandKitId ? repos.getBrandKit(spaceId, project.brandKitId) : Promise.resolve(null), + deps.profiles(), + ]); + return { + project, + kit: kit?.kit ?? null, + assets: new Map(assetRows.map((asset) => [asset.id, asset])), + segments, + faceTracks, + people, + markIds: new Set(marks.map((mark) => mark.id)), + marks: marks.map((mark) => ({ + id: mark.id, + treatment: mark.treatment, + assetId: mark.assetId, + personId: mark.personId, + })), + profiles: new Map( + [...profileSet.profiles.entries()].map(([id, entry]) => [id, entry.profile]), + ), + }; +} + +function kitArtifactIds(kit: BrandKit | null): string[] { + if (!kit) return []; + return [ + ...kit.fonts.map((font) => font.artifactId), + ...kit.logoArtifactIds, + ...(kit.lutArtifactId ? [kit.lutArtifactId] : []), + ...(kit.watermark?.artifactId ? [kit.watermark.artifactId] : []), + ...kit.musicBeds.map((bed) => bed.artifactId), + ]; +} + +export function buildEdlContext( + deps: Pick, + ctx: PlanContext, + profile: PlatformProfile, +): EdlContext { + return { + profile, + brandKit: ctx.kit, + assets: new Map( + [...ctx.assets.values()].map((asset) => [ + asset.id, + { + kind: asset.kind as "video" | "photo" | "live_photo", + durationMs: asset.durationMs, + width: asset.width, + height: asset.height, + }, + ]), + ), + artifactIds: new Set(kitArtifactIds(ctx.kit)), + music: new Map( + (ctx.kit?.musicBeds ?? []).map((bed) => [bed.artifactId, { commercialUse: bed.commercialUse }]), + ), + markIds: ctx.markIds, + personIds: new Set(ctx.people.map((person) => person.id)), + faceTrackIds: new Set(ctx.faceTracks.map((track) => track.id)), + templateIds: new Set(deps.motion.templates().map((template) => template.id)), + paidOnly: ctx.segments + .filter((segment) => segment.paidOnly) + .map((segment) => ({ + assetId: segment.assetId, + startMs: segment.startMs, + endMs: segment.endMs, + })), + goal: ctx.project.goal, + }; +} + +function segmentFeatures(segment: MediaSegment): SegmentFeatures { + const features: SegmentFeatures = { ...segment.scores }; + if (features.speechHook === undefined && segment.transcriptExcerpt) { + features.speechHook = speechHookScore(segment.transcriptExcerpt); + } + return features; +} + +function scoredInputs(ctx: PlanContext): ScoredSegmentInput[] { + return ctx.segments.map((segment) => ({ + id: segment.id, + assetId: segment.assetId, + startMs: segment.startMs, + endMs: segment.endMs, + capturedAt: ctx.assets.get(segment.assetId)?.capturedAt ?? null, + features: segmentFeatures(segment), + paidOnly: segment.paidOnly, + transcript: segment.transcriptExcerpt, + })); +} + +/** Scale features so the model's order counts as much as the deterministic score. */ +export function applyRanking( + segments: ScoredSegmentInput[], + order: string[], +): ScoredSegmentInput[] { + const rank = new Map(order.map((id, index) => [id, index])); + return segments.map((segment) => { + // Paid-only segments are never used; they only mark the payoff, so they keep their score. + if (segment.paidOnly) return segment; + const r = rank.get(segment.id); + const factor = r === undefined ? 0.5 : 1 - r / (2 * order.length); + const features = Object.fromEntries( + Object.entries(segment.features).map(([feature, value]) => [feature, (value ?? 0) * factor]), + ) as SegmentFeatures; + return { ...segment, features }; + }); +} + +async function classifierFrames( + deps: MediaServiceDeps, + spaceId: string, + assetIds: Set, +): Promise> { + const analyses = await deps.repos.latestAnalysesByKind(spaceId, "classifier"); + const out = new Map(); + for (const [assetId, row] of analyses) { + if (!assetIds.has(assetId)) continue; + let data: unknown = row.data; + if ((data === null || data === undefined) && row.dataPath) { + try { + data = JSON.parse(await readFile(await resolveInside(deps.mediaRoot, row.dataPath), "utf8")); + } catch { + continue; + } + } + const frames = (data as { frames?: unknown } | null)?.frames; + if (!Array.isArray(frames)) continue; + out.set( + assetId, + frames.filter( + (frame): frame is ClassifierFrames[number] => + typeof frame?.tMs === "number" && typeof frame.scores === "object" && frame.scores !== null, + ), + ); + } + return out; +} + +/** Fail closed: a segment without classifier frames in its range does not pass. */ +function classifierPasses( + frames: ClassifierFrames | undefined, + segment: { startMs: number; endMs: number }, + profile: PlatformProfile, +): boolean { + const inRange = (frames ?? []).filter( + (frame) => frame.tMs >= segment.startMs && frame.tMs <= segment.endMs, + ); + if (inRange.length === 0) return false; + return inRange.every((frame) => + Object.entries(profile.content.thresholds).every(([category, threshold]) => { + const score = frame.scores[category as ContentCategory]; + return score === undefined || threshold === undefined || score <= threshold; + }), + ); +} + +function protectedPersonIds(ctx: PlanContext): Set { + return new Set( + ctx.people.filter((person) => person.anonymizeByDefault !== "none").map((p) => p.id), + ); +} + +function showsProtectedFace( + ctx: PlanContext, + protectedIds: Set, + segment: { assetId: string; startMs: number; endMs: number }, +): boolean { + if (protectedIds.size === 0) return false; + return ctx.faceTracks.some( + (track) => + track.assetId === segment.assetId && + track.startMs < segment.endMs && + track.endMs > segment.startMs && + (track.personId === null || protectedIds.has(track.personId)), + ); +} + +interface Judgement { + order: string[]; + reasons: Map; +} + +async function judgeCandidates( + deps: MediaServiceDeps, + actor: MediaActor, + ctx: PlanContext, + profile: PlatformProfile, + segments: ScoredSegmentInput[], + weights: TeaserWeights, +): Promise { + const top = segments + .filter((segment) => !segment.paidOnly && segment.endMs > segment.startMs) + .map((segment) => ({ segment, score: scoreSegment(segment.features, weights) })) + .sort((a, b) => b.score - a.score || a.segment.id.localeCompare(b.segment.id)) + .slice(0, JUDGE_TOP_K); + if (top.length < 2) return null; + const classifier = await classifierFrames( + deps, + actor.spaceId, + new Set(top.map((c) => c.segment.assetId)), + ); + const protectedIds = protectedPersonIds(ctx); + const eligible = top.filter( + ({ segment }) => + classifierPasses(classifier.get(segment.assetId), segment, profile) && + !showsProtectedFace(ctx, protectedIds, segment), + ); + if (eligible.length < 2) return null; + + const workRelative = path.join(actor.spaceId, "tmp", `judge-${crypto.randomUUID()}`); + const workDir = await resolveInside(deps.mediaRoot, workRelative); + await mkdir(workDir, { recursive: true }); + try { + const images: AgentInputImage[] = []; + const sent: typeof eligible = []; + for (const candidate of eligible) { + const asset = ctx.assets.get(candidate.segment.assetId); + const isVideo = asset?.kind === "video"; + const source = isVideo ? asset?.proxyPath : (asset?.thumbPath ?? asset?.proxyPath); + if (!source) continue; + const outPath = path.join(workDir, `${images.length}.png`); + try { + await deps.renderer.extractFrame({ + path: await resolveInside(deps.mediaRoot, source), + tMs: isVideo + ? Math.round((candidate.segment.startMs + candidate.segment.endMs) / 2) + : 0, + outPath, + width: JUDGE_FRAME_WIDTH, + }); + images.push({ + name: `${candidate.segment.id}.png`, + mimeType: "image/png", + data: new Uint8Array(await readFile(outPath)), + }); + sent.push(candidate); + } catch (error) { + getLogger().warn("media judge frame failed", { + reason: error instanceof Error ? error.message : String(error), + }); + } + } + if (sent.length < 2) return null; + const lines = sent.map(({ segment, score }, index) => { + const transcript = (segment.transcript ?? "").replace(/\s+/g, " ").trim(); + return [ + `${index + 1}. id=${segment.id} image=${segment.id}.png`, + `duration=${segment.endMs - segment.startMs}ms score=${score.toFixed(3)}`, + transcript ? `transcript="${transcript.slice(0, TRANSCRIPT_EXCERPT_CHARS)}"` : "", + ] + .filter(Boolean) + .join(" "); + }); + const reply = await deps.model.generate({ + spaceId: actor.spaceId, + userId: actor.userId, + instructions: [ + "You rank candidate segments for a short promotional video.", + `Goal: ${ctx.project.goal}. Platform: ${profile.name}. The first ${profile.duration.hookWindowMs} ms must hook the viewer.`, + "Judge only what the frames and transcripts show. Prefer clear, engaging openers that keep the payoff hidden.", + 'Reply with one JSON object: {"ranking":[{"segmentId":"...","reason":"..."}]}, best first, using only the given ids.', + ].join("\n"), + prompt: `Candidates:\n${lines.join("\n")}`, + images, + schema: JudgeReplySchema, + }); + if (!reply) return null; + const ids = new Set(sent.map(({ segment }) => segment.id)); + const order: string[] = []; + const reasons = new Map(); + for (const entry of reply.ranking) { + if (!ids.has(entry.segmentId) || reasons.has(entry.segmentId)) continue; + order.push(entry.segmentId); + reasons.set(entry.segmentId, entry.reason.replace(/\s+/g, " ").trim()); + } + return order.length > 0 ? { order, reasons } : null; + } finally { + await rm(workDir, { recursive: true, force: true }); + } +} + +function rationaleFor(input: { + edl: Edl; + profile: PlatformProfile; + ctx: PlanContext; + learned: boolean; + judgement: Judgement | null; +}): string { + const { edl, profile, ctx } = input; + const lines = [ + `Automatic ${ctx.project.goal} plan for ${profile.name}.`, + input.learned + ? "Weights learned from this space's published results." + : `Default ${ctx.project.goal} weights.`, + ]; + const first = [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs)[0]; + if (first) { + lines.push( + `Hook: clip ${first.id}${first.segmentId ? ` (segment ${first.segmentId})` : ""} starts at ${introMs(edl) + first.atMs} ms, inside the ${profile.duration.hookWindowMs} ms hook window.`, + ); + } + if (edl.intro) lines.push("Brand intro opens the video."); + else if (ctx.kit?.intro) lines.push("Brand intro left out so the hook comes first."); + if (ctx.segments.some((segment) => segment.paidOnly)) lines.push("Paid-only segments excluded."); + if (input.judgement) { + const reasons = input.judgement.order + .slice(0, 3) + .map((id) => `${id}: ${input.judgement?.reasons.get(id) ?? ""}`.trim()); + lines.push(`Model ranking applied (${input.judgement.order.length} ranked): ${reasons.join("; ")}`); + } + return lines.join("\n").slice(0, RATIONALE_MAX_CHARS); +} + +function requireProfile(ctx: PlanContext, profileId: string): PlatformProfile { + if (!ctx.project.profileIds.includes(profileId)) { + throw new InvalidMediaInputError(`Profile ${profileId} is not part of this project`); + } + const profile = ctx.profiles.get(profileId); + if (!profile) throw new InvalidMediaInputError(`Unknown platform profile ${profileId}`); + return profile; +} + +/** First cut per profile, validated and stored as a new plan version by the agent. */ +export async function autoPlan( + deps: MediaServiceDeps, + actor: MediaActor, + input: { projectId: string; profileIds?: string[] }, +): Promise { + const ctx = await loadPlanContext(deps, actor, input.projectId); + const profileIds = [...new Set(input.profileIds ?? ctx.project.profileIds)]; + const profiles = profileIds.map((id) => requireProfile(ctx, id)); + const model = await deps.repos.getTeaserModel(actor.spaceId); + const learned = Boolean(model && model.sampleCount >= TEASER_LEARNING_MIN_SAMPLES); + const weights = + learned && model ? model.weights : defaultTeaserWeights(ctx.project.goal); + const base = scoredInputs(ctx); + const assets = new Map( + [...ctx.assets.values()].map((asset) => [ + asset.id, + { + durationMs: asset.durationMs, + width: asset.width, + height: asset.height, + hasSpeech: ctx.segments.some( + (segment) => segment.assetId === asset.id && Boolean(segment.transcriptExcerpt?.trim()), + ), + }, + ]), + ); + const projectPeople = new Set( + ctx.faceTracks.flatMap((track) => (track.personId ? [track.personId] : [])), + ); + const people = ctx.people + .filter((person) => projectPeople.has(person.id)) + .map((person) => ({ id: person.id, anonymizeByDefault: person.anonymizeByDefault })); + const endCardCta = ctx.kit?.endCard?.params.cta; + + const plans: MediaEditPlan[] = []; + for (const profile of profiles) { + const judgement = RANKING_GOALS.has(ctx.project.goal) + ? await judgeCandidates(deps, actor, ctx, profile, base, weights) + : null; + const segments = judgement ? applyRanking(base, judgement.order) : base; + const edl = autoPlanEdl({ + goal: ctx.project.goal, + profile, + brandKit: ctx.kit, + segments, + weights, + assets, + people, + marks: ctx.marks, + ctaText: typeof endCardCta === "string" ? endCardCta : null, + }); + const validation = validateEdl(edl, buildEdlContext(deps, ctx, profile)); + if (!validation.ok) { + throw new InvalidMediaInputError( + `Automatic plan for ${profile.id} is invalid: ${validation.errors + .map((error) => `${error.path} ${error.message}`) + .join("; ")}`, + ); + } + plans.push( + await deps.repos.createPlanVersion(actor.spaceId, { + projectId: ctx.project.id, + profileId: profile.id, + edl: validation.edl, + createdBy: "agent", + rationale: rationaleFor({ edl: validation.edl, profile, ctx, learned, judgement }), + }), + ); + } + return plans; +} + +export async function submitPlan( + deps: MediaServiceDeps, + actor: MediaActor, + input: { projectId: string; edl: unknown; rationale: string | null; createdBy: "agent" | "user" }, +): Promise<{ ok: true; plan: MediaEditPlan } | { ok: false; errors: EdlError[] }> { + const ctx = await loadPlanContext(deps, actor, input.projectId); + const profileId = (input.edl as { output?: { profileId?: unknown } } | null)?.output?.profileId; + const profile = + typeof profileId === "string" && ctx.project.profileIds.includes(profileId) + ? ctx.profiles.get(profileId) + : undefined; + if (!profile) { + return { + ok: false, + errors: [ + { + code: "profile_mismatch", + path: "output.profileId", + message: `plan must target one of the project's profiles: ${ctx.project.profileIds.join(", ")}`, + }, + ], + }; + } + const validation = validateEdl(input.edl, buildEdlContext(deps, ctx, profile)); + if (!validation.ok) return { ok: false, errors: validation.errors }; + const plan = await deps.repos.createPlanVersion(actor.spaceId, { + projectId: ctx.project.id, + profileId: profile.id, + edl: validation.edl, + createdBy: input.createdBy, + rationale: input.rationale, + }); + return { ok: true, plan }; +} diff --git a/packages/adapters/src/media-marketing-thumbnails.ts b/packages/adapters/src/media-marketing-thumbnails.ts new file mode 100644 index 0000000..213e9dd --- /dev/null +++ b/packages/adapters/src/media-marketing-thumbnails.ts @@ -0,0 +1,350 @@ +import { access, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { BrandKit, ContentCategory, Edl, MediaRenderOutput, PlatformProfile } from "@rakazo/contracts"; +import { MediaStateConflictError } from "@rakazo/db"; +import { + clipLengthMs, + escapeFilterText, + escapeFilterValue, + introMs, + locateOutputTime, + mainProgramMs, + rankThumbnailFrames, +} from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import sharp from "sharp"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; +import type { MediaToolPaths } from "./media-folder-source-probe.js"; +import { runMediaTool } from "./media-folder-source-probe.js"; +import type { StoredRenderOutput } from "./media-marketing-files.js"; +import { + materializeArtifact, + publicRenderOutput, + renderOutputPath, + storeMediaArtifact, + storedRenderOutputs, +} from "./media-marketing-files.js"; + +/* + * Thumbnail and poster variants for a finished render (spec 10.1). + * + * Candidates are 24 evenly spaced frames of the main program plus the frame + * just after every cut. Each is decoded small and scored: sharpness is the + * variance of a 4-neighbour Laplacian, contrast the luminance standard + * deviation (both relative to the best candidate), thirds the gradient energy + * near the rule-of-thirds points against the frame average, and face 1 when a + * stored face track of an unprotected person covers the source time. + * rankThumbnailFrames orders them; the best frames are extracted at full size + * as PNG (no EXIF or text chunks) with the plan title drawn in the brand + * colours inside the safe zone. Every variant passes the content thresholds of + * the profile and, when the project protects a person, must show no detectable + * face; failing frames are dropped and the next candidate takes their place. + */ + +export const THUMBNAIL_CANDIDATES = 24; +const MAX_VARIANTS = 10; +const ANALYSIS_WIDTH = 160; +const MAX_THUMB_WIDTH = 1920; +const CUT_OFFSET_MS = 200; +const TITLE_MAX_CHARS = 80; +const DEJAVU_BOLD = [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", + "/usr/share/fonts/dejavu/DejaVuSans-Bold.ttf", + "/usr/share/fonts/TTF/DejaVuSans-Bold.ttf", +]; + +export function thumbnailVariantName(index: number): string { + return `thumb-${String.fromCharCode(97 + index)}`; +} + +export interface FrameMeasure { + /** Variance of the 4-neighbour Laplacian over the grey frame. */ + laplacianVariance: number; + /** Standard deviation of the grey levels. */ + contrast: number; + /** 0..1: gradient energy near the third intersections relative to the frame. */ + thirds: number; +} + +/** Sharpness, contrast and thirds of a packed 8-bit grey frame. */ +export function measureGrayFrame(data: Uint8Array, width: number, height: number): FrameMeasure { + let sum = 0; + let sumSq = 0; + for (let i = 0; i < width * height; i++) { + const value = data[i] ?? 0; + sum += value; + sumSq += value * value; + } + const pixels = Math.max(1, width * height); + const mean = sum / pixels; + const contrast = Math.sqrt(Math.max(0, sumSq / pixels - mean * mean)); + + let lapSum = 0; + let lapSq = 0; + let lapCount = 0; + let energyAll = 0; + let energyThirds = 0; + let thirdsCount = 0; + const radiusX = width / 8; + const radiusY = height / 8; + const points = [1 / 3, 2 / 3].flatMap((fx) => + [1 / 3, 2 / 3].map((fy) => [fx * width, fy * height] as const), + ); + for (let y = 1; y < height - 1; y++) { + for (let x = 1; x < width - 1; x++) { + const at = (dx: number, dy: number) => data[(y + dy) * width + x + dx] ?? 0; + const center = at(0, 0); + const lap = 4 * center - at(-1, 0) - at(1, 0) - at(0, -1) - at(0, 1); + lapSum += lap; + lapSq += lap * lap; + lapCount += 1; + const gradient = Math.abs(at(1, 0) - at(-1, 0)) + Math.abs(at(0, 1) - at(0, -1)); + energyAll += gradient; + if (points.some(([px, py]) => Math.abs(x - px) <= radiusX && Math.abs(y - py) <= radiusY)) { + energyThirds += gradient; + thirdsCount += 1; + } + } + } + const lapMean = lapCount ? lapSum / lapCount : 0; + const laplacianVariance = lapCount ? Math.max(0, lapSq / lapCount - lapMean * lapMean) : 0; + const meanAll = lapCount ? energyAll / lapCount : 0; + const meanThirds = thirdsCount ? energyThirds / thirdsCount : 0; + const thirds = meanAll > 0 ? Math.min(1, meanThirds / meanAll / 2) : 0; + return { laplacianVariance, contrast, thirds }; +} + +/** Candidate output times: evenly spaced plus just after each cut, inside the main program. */ +export function thumbnailCandidateTimes(edl: Edl, durationMs: number): number[] { + const start = introMs(edl); + const end = Math.min(durationMs, start + mainProgramMs(edl)); + if (end <= start) return []; + const span = end - start; + const times = new Set(); + for (let i = 0; i < THUMBNAIL_CANDIDATES; i++) { + times.add(Math.round(start + ((i + 0.5) * span) / THUMBNAIL_CANDIDATES)); + } + for (const clip of edl.tracks.video) { + const t = start + clip.atMs + Math.min(CUT_OFFSET_MS, clipLengthMs(clip) / 2); + if (t > start && t < end) times.add(Math.round(t)); + } + return [...times].sort((a, b) => a - b); +} + +async function existingFile(candidates: string[]): Promise { + for (const file of candidates) { + try { + await access(file); + return file; + } catch { + // try the next location + } + } + return null; +} + +function ffmpegColor(hex: string | undefined, fallback: string): string { + return hex && /^#[0-9a-fA-F]{6}$/.test(hex) ? `0x${hex.slice(1)}` : fallback; +} + +/** drawtext title band inside the top safe zone; every value is escaped. */ +export function titleOverlayFilter(input: { + text: string; + width: number; + height: number; + profile: PlatformProfile; + kit: BrandKit | null; + fontFile: string | null; +}): string { + const text = input.text.replace(/\s+/g, " ").trim().slice(0, TITLE_MAX_CHARS); + const fontSize = Math.max( + 12, + Math.round(Math.min(input.height / 14, (input.width * 0.9) / Math.max(1, text.length * 0.6))), + ); + const y = Math.round(input.height * (input.profile.safeZones.topPct / 100 + 0.04)); + const font = input.fontFile + ? `fontfile=${escapeFilterValue(input.fontFile)}` + : `font=${escapeFilterValue("DejaVu Sans")}`; + return [ + `drawtext=${font}`, + `text=${escapeFilterText(text)}`, + `fontsize=${fontSize}`, + `fontcolor=${ffmpegColor(input.kit?.colors.text, "white")}`, + "box=1", + `boxcolor=${ffmpegColor(input.kit?.colors.background, "black")}@0.6`, + `boxborderw=${Math.max(4, Math.round(fontSize / 3))}`, + "x=(w-text_w)/2", + `y=${y}`, + ].join(":"); +} + +function exceedsThresholds( + frames: Array<{ scores: Partial> }>, + profile: PlatformProfile, +): boolean { + return frames.some((frame) => + Object.entries(profile.content.thresholds).some(([category, threshold]) => { + const score = frame.scores[category as ContentCategory]; + return score !== undefined && threshold !== undefined && score > threshold; + }), + ); +} + +export async function createThumbnails( + deps: MediaServiceDeps, + actor: MediaActor, + input: { renderId: string; count: number }, + tools: MediaToolPaths = {}, +): Promise { + const { spaceId } = actor; + const row = await deps.repos.getRenderRecord(spaceId, input.renderId); + if (row.status !== "done") throw new MediaStateConflictError("Thumbnails need a finished render"); + const outputs = storedRenderOutputs(row.outputs); + const video = outputs.find((output) => output.kind === "video"); + if (!video) throw new MediaStateConflictError("Render has no video output"); + const videoPath = await renderOutputPath(deps, spaceId, row.id, video); + const [plan, profileSet, project] = await Promise.all([ + deps.repos.getPlan(spaceId, row.planId), + deps.profiles(), + deps.repos.getProjectRecord(spaceId, row.projectId), + ]); + const profile = profileSet.profiles.get(row.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${row.profileId} is not available`); + const edl = plan.edl; + const kit = project.brandKitId + ? (await deps.repos.getBrandKit(spaceId, project.brandKitId)).kit + : null; + const stored = row.ffprobe as { durationMs?: unknown } | null; + const durationMs = + typeof stored?.durationMs === "number" + ? stored.durationMs + : (await deps.renderer.probe(videoPath)).durationMs; + + const assetIds = [...new Set(edl.tracks.video.map((clip) => clip.assetId))]; + const tracks = await deps.repos.faceTrackRecords(spaceId, assetIds); + const personIds = [...new Set(tracks.flatMap((t) => (t.personId ? [t.personId] : [])))]; + const people = personIds.length ? await deps.repos.listPersonRecords(spaceId, personIds) : []; + const protectedIds = new Set( + people.filter((person) => person.anonymizeByDefault !== "none").map((person) => person.id), + ); + const protectsFaces = protectedIds.size > 0 || edl.redactions.faces.length > 0; + + const relativeDir = path.join(spaceId, "renders", row.id, "thumbs"); + const dir = await resolveInside(deps.mediaRoot, relativeDir); + const candidateDir = path.join(dir, "candidates"); + await rm(candidateDir, { recursive: true, force: true }); + await mkdir(candidateDir, { recursive: true }); + try { + const measured: Array<{ tMs: number } & FrameMeasure & { face: number }> = []; + for (const tMs of thumbnailCandidateTimes(edl, durationMs)) { + const file = path.join(candidateDir, `${tMs}.png`); + await deps.renderer.extractFrame({ path: videoPath, tMs, outPath: file, width: ANALYSIS_WIDTH }); + const { data, info } = await sharp(file).greyscale().raw().toBuffer({ resolveWithObject: true }); + const at = locateOutputTime(edl, tMs); + const face = + at && + tracks.some( + (track) => + track.assetId === at.assetId && + track.startMs <= at.sourceMs && + track.endMs >= at.sourceMs && + track.personId !== null && + !protectedIds.has(track.personId), + ) + ? 1 + : 0; + measured.push({ tMs, face, ...measureGrayFrame(data, info.width, info.height) }); + } + const maxSharp = Math.max(1e-9, ...measured.map((m) => m.laplacianVariance)); + const maxContrast = Math.max(1e-9, ...measured.map((m) => m.contrast)); + const order = rankThumbnailFrames( + measured.map((m) => ({ + tMs: m.tMs, + sharpness: m.laplacianVariance / maxSharp, + face: m.face, + thirds: m.thirds, + contrast: m.contrast / maxContrast, + })), + measured.length, + ); + + const title = edl.metadata.title?.trim() || null; + let fontFile: string | null = null; + if (title) { + const heading = kit?.fonts.find((font) => font.role === "heading") ?? kit?.fonts[0]; + fontFile = heading + ? await materializeArtifact(deps, actor, heading.artifactId, path.join(dir, "fonts")) + : await existingFile(DEJAVU_BOLD); + } + const width = Math.min(edl.output.width, MAX_THUMB_WIDTH); + const height = Math.round((width * edl.output.height) / edl.output.width / 2) * 2; + const wanted = Math.max(0, Math.min(MAX_VARIANTS, Math.floor(input.count))); + const results: StoredRenderOutput[] = []; + for (const tMs of order) { + if (results.length >= wanted) break; + const name = thumbnailVariantName(results.length); + const out = path.join(dir, `${name}.png`); + const frame = path.join(candidateDir, `full-${tMs}.png`); + await deps.renderer.extractFrame({ path: videoPath, tMs, outPath: frame, width }); + if (title) { + await runMediaTool(tools.ffmpegPath ?? "ffmpeg", [ + "-v", + "error", + "-y", + "-i", + frame, + "-vf", + titleOverlayFilter({ text: title, width, height, profile, kit, fontFile }), + "-frames:v", + "1", + "-map_metadata", + "-1", + "-bitexact", + "-c:v", + "png", + out, + ]); + } else { + await rename(frame, out); + } + if (deps.inference) { + const content = await deps.inference.classifyContent({ + path: out, + sampleFps: 1, + extraTimesMs: [0], + }); + let rejected = exceedsThresholds(content.frames, profile); + if (!rejected && protectsFaces) { + const faces = await deps.inference.detectFaces({ path: out, sampleFps: 1, timesMs: [0] }); + rejected = faces.frames.some((f) => f.faces.length > 0); + } + if (rejected) { + await rm(out, { force: true }); + continue; + } + } + const bytes = await readFile(out); + const artifactId = await storeMediaArtifact(deps, actor, { + name: `${name}.png`, + mimeType: "image/png", + bytes, + }); + results.push({ + kind: "thumbnail", + artifactId, + fileName: `${name}.png`, + sizeBytes: (await stat(out)).size, + path: path.join(relativeDir, `${name}.png`), + }); + } + await deps.repos.updateRender( + spaceId, + row.id, + { outputs: [...outputs.filter((output) => output.kind !== "thumbnail"), ...results] }, + ["done"], + ); + return results.map(publicRenderOutput); + } finally { + await rm(candidateDir, { recursive: true, force: true }); + } +} diff --git a/packages/adapters/src/media-metrics.ts b/packages/adapters/src/media-metrics.ts new file mode 100644 index 0000000..fec5389 --- /dev/null +++ b/packages/adapters/src/media-metrics.ts @@ -0,0 +1,133 @@ +import type { Edl, MediaSegment } from "@rakazo/contracts"; +import { TEASER_FEATURES, TEASER_LEARNING_MIN_SAMPLES } from "@rakazo/contracts"; +import type { NewMediaMetric, TeaserModelRecord } from "@rakazo/db"; +import { InvalidMediaInputError } from "@rakazo/db"; +import type { SegmentFeatures } from "@rakazo/media"; +import { + clipLengthMs, + defaultTeaserWeights, + fitTeaserWeights, + parseMetricsCsv, + speechHookScore, +} from "@rakazo/media"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Results import and teaser learning (spec 8 "Lernen", 10.6). + * + * A CSV import resolves rows through the saved mapping and records them as + * cumulative snapshots: re-importing a report replaces the same + * (publication, date, source) rows, so numbers never double. After every + * import the teaser weights are refit from all publications with views. Each + * publication contributes the clip-length-weighted mean features of the + * segments its plan used, with its latest views and conversions. Below + * TEASER_LEARNING_MIN_SAMPLES publications nothing is stored and plans keep + * the default weights; resetTeaserModel goes back to them. + */ + +export const METRICS_CSV_MAX_CHARS = 5_000_000; + +/** Clip-length-weighted mean of the features of the segments a plan used. */ +export function planFeatures(edl: Edl, segments: MediaSegment[]): SegmentFeatures | null { + const byId = new Map(segments.map((segment) => [segment.id, segment])); + const totals: Record = {}; + let weight = 0; + for (const clip of edl.tracks.video) { + const used = clip.segmentId + ? [byId.get(clip.segmentId)].filter((s): s is MediaSegment => Boolean(s)) + : segments.filter( + (s) => s.assetId === clip.assetId && s.startMs < clip.outMs && s.endMs > clip.inMs, + ); + if (used.length === 0) continue; + const length = clipLengthMs(clip) / used.length; + for (const segment of used) { + const features: SegmentFeatures = { ...segment.scores }; + if (features.speechHook === undefined && segment.transcriptExcerpt) { + features.speechHook = speechHookScore(segment.transcriptExcerpt); + } + for (const feature of TEASER_FEATURES) { + totals[feature] = (totals[feature] ?? 0) + (features[feature] ?? 0) * length; + } + weight += length; + } + } + if (weight <= 0) return null; + return Object.fromEntries( + TEASER_FEATURES.map((feature) => [feature, (totals[feature] ?? 0) / weight]), + ) as SegmentFeatures; +} + +/** Refit and store the space's teaser weights; null while there are too few results. */ +export async function refitTeaserModel( + deps: Pick, + spaceId: string, +): Promise { + const { repos } = deps; + const rows = (await repos.funnel(spaceId)).filter((row) => row.views > 0); + if (rows.length < TEASER_LEARNING_MIN_SAMPLES) return null; + const plans = new Map(); + const samples: Array<{ edl: Edl; views: number; conversions: number }> = []; + for (const row of rows) { + const publication = await repos.getPublication(spaceId, row.publicationId); + const render = await repos.getRender(spaceId, publication.renderId); + let edl = plans.get(render.planId); + if (!edl) { + edl = (await repos.getPlan(spaceId, render.planId)).edl; + plans.set(render.planId, edl); + } + samples.push({ edl, views: row.views, conversions: row.conversions }); + } + const assetIds = [ + ...new Set(samples.flatMap((s) => s.edl.tracks.video.map((clip) => clip.assetId))), + ]; + const segments = await repos.listSegments(spaceId, assetIds); + const usable = samples.flatMap((sample) => { + const features = planFeatures(sample.edl, segments); + return features + ? [{ features, views: sample.views, conversions: Math.min(sample.conversions, sample.views) }] + : []; + }); + const weights = fitTeaserWeights(usable, defaultTeaserWeights("teaser")); + if (!weights) return null; + return repos.putTeaserModel(spaceId, weights, usable.length); +} + +export async function importMetricsCsv( + deps: Pick, + actor: MediaActor, + input: { mappingId: string; csv: string }, +): Promise<{ imported: number; skipped: Array<{ row: number; reason: string }> }> { + if (input.csv.length > METRICS_CSV_MAX_CHARS) { + throw new InvalidMediaInputError("CSV file is too large"); + } + const { spaceId } = actor; + const mapping = await deps.repos.getMetricMapping(spaceId, input.mappingId); + const parsed = parseMetricsCsv(input.csv, mapping); + const keys = await deps.repos.resolvePublicationKeys( + spaceId, + mapping.publicationKey, + parsed.rows.map((row) => row.publicationKey), + ); + const skipped = [...parsed.skipped]; + const metrics: NewMediaMetric[] = []; + for (const row of parsed.rows) { + const publicationId = keys.get(row.publicationKey.trim()); + if (!publicationId) { + skipped.push({ row: row.row, reason: "publication not found" }); + continue; + } + metrics.push({ + publicationId, + observedAt: row.observedAt.toISOString(), + views: row.views, + clicks: row.clicks, + conversions: row.conversions, + revenueCents: row.revenueCents, + currency: row.currency, + source: "csv_import", + }); + } + const imported = await deps.repos.recordMetrics(spaceId, metrics); + if (imported > 0) await refitTeaserModel(deps, spaceId); + return { imported, skipped: skipped.sort((a, b) => a.row - b.row) }; +} diff --git a/packages/adapters/src/media-motion-renderer.ts b/packages/adapters/src/media-motion-renderer.ts new file mode 100644 index 0000000..f43b1d9 --- /dev/null +++ b/packages/adapters/src/media-motion-renderer.ts @@ -0,0 +1,10 @@ +export type { + MotionFrameContext, + MotionTemplate, + ResvgMotionGraphicsRendererOptions, +} from "./media-motion/index.js"; +export { + MOTION_TEMPLATES, + MotionTemplateError, + ResvgMotionGraphicsRenderer, +} from "./media-motion/index.js"; diff --git a/packages/adapters/src/media-motion/easing.ts b/packages/adapters/src/media-motion/easing.ts new file mode 100644 index 0000000..95c42bc --- /dev/null +++ b/packages/adapters/src/media-motion/easing.ts @@ -0,0 +1,56 @@ +/** Easing and timing helpers shared by every motion template. All pure. */ + +export function clamp(value: number, min = 0, max = 1): number { + return Math.min(max, Math.max(min, value)); +} + +export function lerp(from: number, to: number, amount: number): number { + return from + (to - from) * amount; +} + +/** Linear progress 0..1 of `t` through the window [start, end]. */ +export function progress(t: number, start: number, end: number): number { + if (end <= start) return t >= end ? 1 : 0; + return clamp((t - start) / (end - start)); +} + +export function easeInCubic(x: number): number { + return x * x * x; +} + +export function easeOutCubic(x: number): number { + return 1 - (1 - x) ** 3; +} + +export function easeInOutCubic(x: number): number { + return x < 0.5 ? 4 * x * x * x : 1 - (-2 * x + 2) ** 3 / 2; +} + +export function easeOutExpo(x: number): number { + return x >= 1 ? 1 : 1 - 2 ** (-10 * x); +} + +/** Overshoots slightly before settling; for pop-in reveals. */ +export function easeOutBack(x: number): number { + const c1 = 1.70158; + const c3 = c1 + 1; + return 1 + c3 * (x - 1) ** 3 + c1 * (x - 1) ** 2; +} + +/** + * Opacity envelope for a clip of `durationMs`: fades in over `inMs` and out + * over the last `outMs`. + */ +export function fadeEnvelope(t: number, durationMs: number, inMs: number, outMs: number): number { + const fadeIn = inMs > 0 ? easeOutCubic(progress(t, 0, inMs)) : 1; + const fadeOut = outMs > 0 ? 1 - easeInCubic(progress(t, durationMs - outMs, durationMs)) : 1; + return clamp(Math.min(fadeIn, fadeOut)); +} + +/** Deterministic pseudo-random value in [0, 1) for integer seeds. */ +export function hash01(a: number, b = 0): number { + let h = Math.imul(a ^ 0x9e3779b9, 0x85ebca6b) ^ Math.imul(b + 0x632be5ab, 0xc2b2ae35); + h = Math.imul(h ^ (h >>> 15), 0x2c1b3c6d); + h = Math.imul(h ^ (h >>> 12), 0x297a2d39); + return ((h ^ (h >>> 15)) >>> 0) / 4294967296; +} diff --git a/packages/adapters/src/media-motion/index.ts b/packages/adapters/src/media-motion/index.ts new file mode 100644 index 0000000..1161c6b --- /dev/null +++ b/packages/adapters/src/media-motion/index.ts @@ -0,0 +1,308 @@ +/* + * Motion graphics renderer for Media Studio (intros, outros, lower thirds, + * caption previews, end cards). + * + * Technology decision: no Remotion and no headless Chromium. Remotion needs a + * paid company license above a team size and Chromium is a heavy, hard-to- + * sandbox runtime. Instead every template is a pure TypeScript function that + * returns an SVG document for a point in time. Frames are rasterized with + * @resvg/resvg-js (MPL-2.0; resvg itself is MPL-2.0, used unmodified as a + * library, commercially usable) and piped as PNGs into ffmpeg (LGPL/GPL + * depending on the build, invoked as a separate process with an argv array), + * which encodes a QuickTime Animation (qtrle, argb) MOV with alpha. Nothing + * here needs a network or a hosted service. + */ +import type { ChildProcess } from "node:child_process"; +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { readFile, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { MotionGraphicsRenderer, MotionRenderRequest } from "@rakazo/adapter-kit"; +import type { BrandKit, MotionTemplateInfo, MotionTemplateRef } from "@rakazo/contracts"; +import { BrandKitSchema, MotionTemplateRefSchema } from "@rakazo/contracts"; +import { assertInside } from "@rakazo/media/node"; +import type { RenderedImage, ResvgRenderOptions } from "@resvg/resvg-js"; +import { renderAsync } from "@resvg/resvg-js"; +import type { MotionFrameContext, MotionTemplate } from "./template.js"; +import { brandFonts, MotionTemplateError, validateDuration, validateParams } from "./template.js"; +import { captionBoldSocial, captionClean, captionKaraoke } from "./templates/captions.js"; +import { endCardCta, endCardMinimal } from "./templates/end-cards.js"; +import { + introElegantFade, + introGlitch, + introKineticType, + introLogoReveal, +} from "./templates/intros.js"; +import { lowerThirdBar, lowerThirdClean, lowerThirdPill } from "./templates/lower-thirds.js"; +import { outroLogo, outroSubscribe } from "./templates/outros.js"; + +export type { MotionFrameContext, MotionTemplate } from "./template.js"; +export { MotionTemplateError } from "./template.js"; + +export const MOTION_TEMPLATES: readonly MotionTemplate[] = [ + introLogoReveal, + introKineticType, + introGlitch, + introElegantFade, + outroLogo, + outroSubscribe, + lowerThirdClean, + lowerThirdBar, + lowerThirdPill, + captionClean, + captionKaraoke, + captionBoldSocial, + endCardCta, + endCardMinimal, +]; + +export function motionTemplate(id: string): MotionTemplate { + const template = MOTION_TEMPLATES.find((entry) => entry.info.id === id); + if (!template) throw new MotionTemplateError("unknown_template", `unknown template "${id}"`); + return template; +} + +export const MIN_FRAME_SIDE = 16; +export const MAX_FRAME_SIDE = 3840; +export const MAX_FPS = 60; +/** Default cap on width × height × frames: ten seconds of 1080p60. */ +export const DEFAULT_MAX_PIXEL_FRAMES = 1920 * 1080 * 60 * 10; +const MAX_LOGO_BYTES = 5 * 1024 * 1024; +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const FONT_EXTENSIONS = new Set([".ttf", ".otf", ".ttc"]); + +/** + * Read a logo as a data URI. Only PNG and SVG are accepted, and an SVG may not + * declare entities or reference anything outside itself. + */ +export async function loadLogoDataUri(file: string): Promise { + const extension = path.extname(file).toLowerCase(); + if (extension !== ".png" && extension !== ".svg") + throw new MotionTemplateError("invalid_asset", "logos must be PNG or SVG"); + const info = await stat(file); + if (!info.isFile() || info.size > MAX_LOGO_BYTES) + throw new MotionTemplateError("invalid_asset", "logo is not a file or is too large"); + const bytes = await readFile(file); + if (extension === ".png") { + if (!bytes.subarray(0, 8).equals(PNG_SIGNATURE)) + throw new MotionTemplateError("invalid_asset", "logo is not a PNG"); + return `data:image/png;base64,${bytes.toString("base64")}`; + } + const text = bytes.toString("utf8"); + if (!/]/i.test(text)) + throw new MotionTemplateError("invalid_asset", "logo is not an SVG"); + if (/ MAX_FRAME_SIDE) + throw new MotionTemplateError( + "invalid_size", + `frame sides must be ${MIN_FRAME_SIDE}..${MAX_FRAME_SIDE} px`, + ); + } + return { + template, + ctx: { + width: input.width, + height: input.height, + durationMs: ref.durationMs, + kit, + params: validateParams(template.info, ref.params), + logos: input.logos, + fonts: brandFonts(kit), + }, + }; +} + +export interface RasterizeOptions { + fontFiles: string[]; + defaultFontFamily?: string | null; + /** Fall back to system fonts for missing families and glyphs. */ + loadSystemFonts: boolean; +} + +export function rasterizeFrame(svg: string, options: RasterizeOptions): Promise { + const resvgOptions: ResvgRenderOptions = { + font: { + loadSystemFonts: options.loadSystemFonts, + fontFiles: options.fontFiles, + ...(options.defaultFontFamily ? { defaultFontFamily: options.defaultFontFamily } : {}), + }, + logLevel: "off", + }; + return renderAsync(svg, resvgOptions); +} + +export function frameCount(durationMs: number, fps: number): number { + return Math.max(1, Math.round((durationMs * fps) / 1000)); +} + +function abortError(): Error { + const error = new Error("motion render aborted"); + error.name = "AbortError"; + return error; +} + +export interface ResvgMotionGraphicsRendererOptions { + ffmpegPath?: string; + /** When set, every asset path and the output path must lie inside it. */ + root?: string; + maxPixelFrames?: number; +} + +export class ResvgMotionGraphicsRenderer implements MotionGraphicsRenderer { + readonly id = "resvg"; + private readonly ffmpegPath: string; + private readonly maxPixelFrames: number; + + constructor(private readonly options: ResvgMotionGraphicsRendererOptions = {}) { + this.ffmpegPath = options.ffmpegPath ?? "ffmpeg"; + this.maxPixelFrames = options.maxPixelFrames ?? DEFAULT_MAX_PIXEL_FRAMES; + } + + templates(): MotionTemplateInfo[] { + return MOTION_TEMPLATES.map((template) => structuredClone(template.info)); + } + + async render(request: MotionRenderRequest): Promise<{ path: string; hasAlpha: boolean }> { + const { signal } = request; + if (signal?.aborted) throw abortError(); + if (!Number.isFinite(request.fps) || request.fps < 1 || request.fps > MAX_FPS) + throw new MotionTemplateError("invalid_size", `fps must be 1..${MAX_FPS}`); + if (!path.isAbsolute(request.outPath) || path.extname(request.outPath).toLowerCase() !== ".mov") + throw new MotionTemplateError("invalid_asset", "outPath must be an absolute .mov path"); + + const outPath = await this.checkedPath(request.outPath); + const fontFiles: string[] = []; + for (const font of request.assets.fonts) { + const file = await this.checkedPath(font.path); + if (!FONT_EXTENSIONS.has(path.extname(file).toLowerCase()) || !(await stat(file)).isFile()) + throw new MotionTemplateError("invalid_asset", "fonts must be TTF, OTF, or TTC files"); + fontFiles.push(file); + } + const logos: string[] = []; + for (const logo of request.assets.logoPaths) { + logos.push(await loadLogoDataUri(await this.checkedPath(logo))); + } + + const { template, ctx } = buildFrameContext({ + template: request.template, + kit: request.brandKit, + width: request.width, + height: request.height, + logos, + }); + const frames = frameCount(ctx.durationMs, request.fps); + if (ctx.width * ctx.height * frames > this.maxPixelFrames) + throw new MotionTemplateError("invalid_size", "motion clip exceeds the pixel budget"); + + const raster: RasterizeOptions = { + fontFiles, + defaultFontFamily: ctx.fonts.body, + loadSystemFonts: true, + }; + const partial = path.join( + path.dirname(outPath), + `.${path.basename(outPath, ".mov")}.${randomUUID()}.partial.mov`, + ); + const child = spawn( + this.ffmpegPath, + [ + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-f", + "image2pipe", + "-framerate", + String(request.fps), + "-c:v", + "png", + "-i", + "pipe:0", + "-an", + "-c:v", + "qtrle", + "-pix_fmt", + "argb", + "-f", + "mov", + partial, + ], + { stdio: ["pipe", "ignore", "pipe"], shell: false }, + ); + const exited = waitForExit(child); + const kill = () => child.kill("SIGKILL"); + signal?.addEventListener("abort", kill, { once: true }); + try { + const stdin = child.stdin!; + stdin.on("error", () => {}); + for (let index = 0; index < frames; index += 1) { + if (signal?.aborted) throw abortError(); + const svg = template.frame((index * 1000) / request.fps, ctx); + const png = (await rasterizeFrame(svg, raster)).asPng(); + if (signal?.aborted) throw abortError(); + if (!stdin.write(png)) { + const result = await Promise.race([once(stdin, "drain").then(() => null), exited]); + if (result) throw ffmpegError(result); + } + } + stdin.end(); + const result = await exited; + if (signal?.aborted) throw abortError(); + if (result.code !== 0) throw ffmpegError(result); + await rename(partial, outPath); + return { path: outPath, hasAlpha: true }; + } catch (error) { + kill(); + await exited; + await rm(partial, { force: true }); + throw signal?.aborted ? abortError() : error; + } finally { + signal?.removeEventListener("abort", kill); + } + } + + private async checkedPath(file: string): Promise { + if (!path.isAbsolute(file) || file.includes("\0")) + throw new MotionTemplateError("invalid_asset", "asset paths must be absolute"); + return this.options.root ? assertInside(this.options.root, file) : path.resolve(file); + } +} + +interface ExitResult { + code: number | null; + stderr: string; +} + +function waitForExit(child: ChildProcess): Promise { + let stderr = ""; + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => { + stderr = (stderr + chunk).slice(-4000); + }); + return new Promise((resolve) => { + child.once("error", (error) => resolve({ code: -1, stderr: error.message })); + child.once("close", (code) => resolve({ code, stderr })); + }); +} + +function ffmpegError(result: ExitResult): Error { + return new Error(`ffmpeg failed (${result.code}): ${result.stderr.trim()}`); +} diff --git a/packages/adapters/src/media-motion/media-motion.test.ts b/packages/adapters/src/media-motion/media-motion.test.ts new file mode 100644 index 0000000..edb530a --- /dev/null +++ b/packages/adapters/src/media-motion/media-motion.test.ts @@ -0,0 +1,441 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { BrandKit, MotionTemplateRef } from "@rakazo/contracts"; +import { MotionTemplateInfoSchema } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { MotionTemplate } from "./index.js"; +import { + buildFrameContext, + loadLogoDataUri, + MOTION_TEMPLATES, + MotionTemplateError, + motionTemplate, + ResvgMotionGraphicsRenderer, + rasterizeFrame, +} from "./index.js"; +import { GLITCH_BURST_INTERVAL_MS } from "./templates/intros.js"; + +const DEJAVU_DIR = "/usr/share/fonts/truetype/dejavu"; +const DEJAVU = ["DejaVuSans.ttf", "DejaVuSans-Bold.ttf", "DejaVuSerif.ttf"].map((name) => + path.join(DEJAVU_DIR, name), +); +const hasDejaVu = DEJAVU.every((file) => existsSync(file)); +const hasFfmpeg = (() => { + try { + execFileSync("ffmpeg", ["-hide_banner", "-encoders"], { stdio: "pipe" }); + execFileSync("ffprobe", ["-version"], { stdio: "pipe" }); + return true; + } catch { + return false; + } +})(); + +const LOGO_SVG = + '' + + '' + + ''; + +const kit: BrandKit = { + name: "Test Brand", + colors: { + primary: "#2255cc", + secondary: "#9999aa", + accent: "#ff8800", + text: "#ffffff", + background: "#101820", + }, + fonts: [{ family: "DejaVu Sans", artifactId: "font-1", role: "heading" }], + logoArtifactIds: ["logo-1"], + captionPresets: [], + musicBeds: [], +}; + +/** Parameters that exercise every template with all its text slots filled. */ +const SAMPLE_PARAMS: Record = { + intro_logo_reveal: { tagline: "Make it simple" }, + intro_kinetic_type: { title: "Bold ideas move fast", tagline: "Since 2026" }, + intro_glitch: { tagline: "Energy" }, + intro_elegant_fade: { tagline: "Crafted slowly" }, + outro_logo: { message: "Thanks" }, + outro_subscribe: { cta: "Subscribe", handle: "@example" }, + lt_clean: { title: "Alex Example", subtitle: "Host" }, + lt_bar: { title: "Alex Example", subtitle: "Host", barColor: "accent" }, + lt_pill: { title: "Alex Example", subtitle: "Host" }, + caption_clean: { text: "A caption in the clean style" }, + caption_karaoke: { text: "A caption in the karaoke style" }, + caption_bold_social: { text: "Bold social caption" }, + endcard_cta: { headline: "Get the full video", cta: "Watch now", price: "Only 9 today" }, + endcard_minimal: { url: "example.com" }, +}; + +let dir: string; +let logoPath: string; +let logo: string; + +beforeAll(async () => { + dir = await mkdtemp(path.join(tmpdir(), "rakazo-motion-")); + logoPath = path.join(dir, "logo.svg"); + await writeFile(logoPath, LOGO_SVG); + logo = await loadLogoDataUri(logoPath); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +function context( + template: MotionTemplate, + overrides: { + params?: MotionTemplateRef["params"]; + durationMs?: number; + size?: [number, number]; + } = {}, +) { + const [width, height] = overrides.size ?? [640, 360]; + return buildFrameContext({ + template: { + templateId: template.info.id, + params: overrides.params ?? SAMPLE_PARAMS[template.info.id] ?? {}, + durationMs: overrides.durationMs ?? template.info.minDurationMs, + }, + kit, + width, + height, + logos: [logo], + }).ctx; +} + +const raster = { fontFiles: DEJAVU, defaultFontFamily: "DejaVu Sans", loadSystemFonts: false }; + +/** Mean relative luminance of an RGBA frame composited over a solid backdrop. */ +function meanLuminance(pixels: Buffer, backdrop: 0 | 1): number { + let sum = 0; + const count = pixels.length / 4; + for (let index = 0; index < pixels.length; index += 4) { + const alpha = pixels[index + 3]! / 255; + const channel = (offset: number) => { + const value = (pixels[index + offset]! / 255) * alpha + backdrop * (1 - alpha); + return value <= 0.03928 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4; + }; + sum += 0.2126 * channel(0) + 0.7152 * channel(1) + 0.0722 * channel(2); + } + return sum / count; +} + +/** + * Count brightness reversals of at least 10% relative luminance (the WCAG 2.3.1 + * flash threshold) per one-second window and return the worst window. A flash + * is a pair of opposing changes, so a steady fade counts as none; counting each + * reversal on its own is stricter than WCAG's three flashes per second. + */ +function maxReversalsPerSecond(series: number[], fps: number): number { + const transitions: number[] = []; + let direction = 0; + let extreme = series[0] ?? 0; + series.forEach((value, index) => { + const delta = value - extreme; + if (direction !== 0 && Math.sign(delta) === direction) { + extreme = value; + } else if (Math.abs(delta) >= 0.1) { + if (direction !== 0) transitions.push(index / fps); + direction = Math.sign(delta); + extreme = value; + } + }); + let worst = 0; + for (const start of transitions) { + worst = Math.max(worst, transitions.filter((at) => at >= start && at < start + 1).length); + } + return worst; +} + +describe("motion templates", () => { + it("ships the required set with valid, unique descriptors", () => { + const ids = MOTION_TEMPLATES.map((template) => template.info.id); + expect(new Set(ids).size).toBe(ids.length); + for (const template of MOTION_TEMPLATES) MotionTemplateInfoSchema.parse(template.info); + const count = (kind: string) => + MOTION_TEMPLATES.filter((template) => template.info.kind === kind).length; + expect([ + count("intro"), + count("outro"), + count("lower_third"), + count("caption"), + count("end_card"), + ]).toEqual([4, 2, 3, 3, 2]); + }); + + it.each(MOTION_TEMPLATES.map((template) => [template.info.id, template] as const))( + "%s renders its first, middle and last frame", + async (_id, template) => { + const ctx = context(template); + for (const t of [0, ctx.durationMs / 2, ctx.durationMs]) { + const svg = template.frame(t, ctx); + expect(svg.startsWith(" { + const allowed = new Set(Object.values(kit.colors).map((color) => color.toLowerCase())); + for (const template of MOTION_TEMPLATES) { + const ctx = context(template); + const svg = template.frame(ctx.durationMs / 2, ctx); + const colors = [...svg.matchAll(/(?:fill|stroke)="([^"]+)"/g)].map((match) => match[1]!); + for (const color of colors) { + if (color === "none" || color.startsWith("url(")) continue; + expect(allowed.has(color.toLowerCase()), `${template.info.id} uses ${color}`).toBe(true); + } + } + }); + + it.skipIf(!hasDejaVu)("renders deterministic middle frames", async () => { + const hashes: Record = {}; + for (const template of MOTION_TEMPLATES) { + const ctx = context(template); + const image = await rasterizeFrame(template.frame(ctx.durationMs / 2, ctx), raster); + hashes[template.info.id] = createHash("sha256") + .update(image.pixels) + .digest("hex") + .slice(0, 16); + } + expect(hashes).toMatchInlineSnapshot(` + { + "caption_bold_social": "4a132766b59cdf12", + "caption_clean": "6194d5224d8e97ec", + "caption_karaoke": "f2db9bc313e4ccea", + "endcard_cta": "943fa754c9a30803", + "endcard_minimal": "be2b9f833895bc36", + "intro_elegant_fade": "8f8a98930994243f", + "intro_glitch": "35074c7ad30d26bb", + "intro_kinetic_type": "52129b6c4f5581e3", + "intro_logo_reveal": "2baf101db044941b", + "lt_bar": "a13184996d2ffc2e", + "lt_clean": "ca6a4239283e111e", + "lt_pill": "30a7342e49735d69", + "outro_logo": "84b1cd23213f8a2e", + "outro_subscribe": "18733abc1b0f1296", + } + `); + }); + + it("never changes full-frame brightness more than three times a second in the glitch intro", async () => { + const template = motionTemplate("intro_glitch"); + const ctx = context(template, { durationMs: 4000, size: [160, 90] }); + const fps = 25; + const dark: number[] = []; + const light: number[] = []; + for (let index = 0; index <= (ctx.durationMs * fps) / 1000; index += 1) { + const image = await rasterizeFrame(template.frame((index * 1000) / fps, ctx), { + fontFiles: [], + loadSystemFonts: true, + }); + dark.push(meanLuminance(image.pixels, 0)); + light.push(meanLuminance(image.pixels, 1)); + } + expect(1000 / GLITCH_BURST_INTERVAL_MS).toBeLessThanOrEqual(3); + expect(maxReversalsPerSecond(dark, fps)).toBeLessThanOrEqual(3); + expect(maxReversalsPerSecond(light, fps)).toBeLessThanOrEqual(3); + }); + + it("detects a frame that does flash too fast", () => { + const flashing = Array.from({ length: 50 }, (_, index) => (index % 5 < 2 ? 0.9 : 0.05)); + expect(maxReversalsPerSecond(flashing, 50)).toBeGreaterThan(3); + }); +}); + +describe("motion params and escaping", () => { + const lowerThird = motionTemplate("lt_clean"); + + it.each([ + [{ title: "x", extra: "y" }, "unknown_param"], + [{ title: 3 }, "invalid_param"], + [{ title: "a\u0007b" }, "invalid_param"], + [{ subtitle: "only" }, "missing_param"], + ] as const)("rejects %j", (params, code) => { + expect(() => context(lowerThird, { params })).toThrow( + expect.objectContaining({ code }) as unknown as Error, + ); + }); + + it("rejects oversized strings, unknown colors, durations, and frame sizes", () => { + expect(() => context(lowerThird, { params: { title: "x".repeat(201) } })).toThrow(); + expect(() => + context(motionTemplate("lt_bar"), { params: { title: "x", barColor: "#ff0000" } }), + ).toThrow(MotionTemplateError); + expect(() => context(lowerThird, { durationMs: 60_000 })).toThrow(MotionTemplateError); + expect(() => context(lowerThird, { size: [8000, 360] })).toThrow(MotionTemplateError); + expect(() => + buildFrameContext({ + template: { templateId: "nope", params: {}, durationMs: 2000 }, + kit, + width: 640, + height: 360, + logos: [], + }), + ).toThrow(MotionTemplateError); + }); + + it("escapes text and keeps hostile input out of the markup", async () => { + const hostile = ` & 'quotes' ]]> now. +", + "id": "captions-ass", + "kind": "write_file", + "path": "/work/render-1/captions.ass", + }, + { + "content": "1 +00:00:00,200 --> 00:00:01,800 +Wait for this! + +2 +00:00:02,100 --> 00:00:03,400 +Here it is -> now. +", + "id": "captions-srt", + "kind": "write_file", + "path": "/work/render-1/captions.srt", + }, + { + "content": "WEBVTT + +00:00:00.200 --> 00:00:01.800 +Wait for this! + +00:00:02.100 --> 00:00:03.400 +Here it <b>is</b> --> now. +", + "id": "captions-vtt", + "kind": "write_file", + "path": "/work/render-1/captions.vtt", + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "5", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-i", + "/brand/watermark.png", + "-filter_complex", + "[0:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,170.82\\,if(lt(t\\,1.5)\\,170.82+(167.81)*(t-0)/1.5\\,if(lt(t\\,2)\\,338.62+(19.97)*(t-1.5)/0.5\\,358.59))):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,1.5)\\,0+(0)*(t-0)/1.5\\,if(lt(t\\,2)\\,0+(0)*(t-1.5)/0.5\\,0))),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,1232\\,if(lt(t\\,2)\\,1232+(0)*(t-0)/2\\,1232)):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,2)\\,0+(0)*(t-0)/2\\,0)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c2];[v_c1][a_c1][v_c2][a_c2]concat=n=2:v=1:a=1[cv1][ca2];[2:v]format=rgba,scale=162:-2,colorchannelmixer=aa=0.6[img3];[cv1][img3]overlay=x=main_w-overlay_w-152:y=214:enable=between(t\\,0\\,4)[ov4];[ov4]ass=filename=/work/render-1/captions.ass:fontsdir=/brand/fonts[cap5];[cap5]format=yuv420p[vout];[ca2]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "20", + "-profile:v", + "high", + "-maxrate", + "8000k", + "-bufsize", + "16000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "192k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-metadata", + "title=Example: teaser", + "-t", + "4", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 4000, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "1.333", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; + +exports[`compileRenderPlan > compiles transitions, music ducking, voice-over and fades 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "draft", + "steps": [ + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "2", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "3", + "-t", + "1", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "4", + "-t", + "1", + "-i", + "/media/asset-b/original.mov", + "-ss", + "0", + "-t", + "8", + "-i", + "/brand/music/bed.wav", + "-ss", + "0", + "-t", + "2", + "-i", + "/work/render-1/voiceover.wav", + "-filter_complex", + "[0:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/0.5,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=4,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,atempo=0.5,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=4,asetpts=PTS-STARTPTS[a_c2];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c3];[2:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c3];[3:v]crop=w=608:h=1080:x=656:y=0,scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c4];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c4];[v_c1][v_c2]xfade=transition=fade:duration=0.5:offset=1.5[xv1];[a_c1][a_c2]acrossfade=d=0.5:c1=tri:c2=tri[xa2];color=c=black:s=1080x1920:r=30,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=0.5,setpts=PTS-STARTPTS[gv3];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=0.5,asetpts=PTS-STARTPTS[ga4];[xv1][xa2][gv3][ga4][v_c3][a_c3][v_c4][a_c4]concat=n=4:v=1:a=1[cv5][ca6];[cv5]format=yuv420p[vout];[4:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,volume=-8dB,afade=t=in:st=0:d=0.5,afade=t=out:st=7:d=1[ac_bed];[5:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,adelay=1000|1000[ac_vo];[ca6][ac_vo]amix=inputs=2:duration=first:normalize=0[speech];[speech]asplit=2[speech_mix][speech_key];[ac_bed]anull[duck_bus];[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked];[speech_mix][ducked]amix=inputs=2:duration=first:normalize=0[mixed];[mixed]loudnorm=I=-16:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "28", + "-profile:v", + "high", + "-maxrate", + "3000k", + "-bufsize", + "6000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "128k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-t", + "8", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 8000, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "2.667", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; + +exports[`compileRenderPlan > compiles voice disguise, metadata stripping, intro/outro and text overlays 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "final", + "steps": [ + { + "fps": 30, + "height": 1920, + "id": "motion-1", + "kind": "motion", + "outPath": "/work/render-1/motion-1.mov", + "template": { + "durationMs": 1000, + "params": {}, + "templateId": "outro-simple", + }, + "width": 1080, + }, + { + "fps": 30, + "height": 1920, + "id": "motion-2", + "kind": "motion", + "outPath": "/work/render-1/motion-2.mov", + "template": { + "durationMs": 1000, + "params": {}, + "templateId": "lower-third-a", + }, + "width": 1080, + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "3", + "-i", + "/media/asset-a/proxy.mp4", + "-i", + "/work/render-1/intro.mov", + "-i", + "/work/render-1/motion-1.mov", + "-i", + "/work/render-1/motion-2.mov", + "-filter_complex", + "[0:v]deshake,crop=w=406:h=720:x=if(lt(t\\,0)\\,373\\,if(lt(t\\,3)\\,373+(768)*(t-0)/3\\,1141)):y=if(lt(t\\,0)\\,180\\,if(lt(t\\,3)\\,180+(0)*(t-0)/3\\,180)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,eq=brightness=0.05:contrast=1:saturation=1.1,colorbalance=rm=0.02:bm=-0.02,lut3d=file=/brand/look\\\\\\'s.cube,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=3,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[a_c1];[a_c1]rubberband=pitch=0.793701:formant=preserved,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[vd1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1.5,setpts=PTS-STARTPTS[intro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1.5,asetpts=PTS-STARTPTS[intro_a];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[outro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[outro_a];[intro_v][intro_a][v_c1][vd1][outro_v][outro_a]concat=n=3:v=1:a=1[prog_v][prog_a];[3:v]format=yuva420p,scale=1080:1920,setpts=PTS-STARTPTS+0/TB[mo2];[prog_v][mo2]overlay=x=0:y=0:eof_action=pass:enable=between(t\\,0\\,1)[ov3];color=c=0xFFAA00:s=1080x10:r=30:d=5.5[pb4];[ov3][pb4]overlay=x=-overlay_w+main_w*min(1\\,(t-1.5)/3):y=1504:enable=between(t\\,1.5\\,4.5)[ov5];[ov5]drawtext=fontfile=/brand/fonts/ExampleSans.ttf:text=Ana\\\\\\'s\\\\: \\[studio\\]\\; 100\\\\\\\\% real:fontsize=77:fontcolor=0xFFFFFF:x=(w-text_w)/2:y=h-text_h-560:box=1:boxcolor=0x112233@0.85:boxborderw=31:enable=between(t\\,2\\,3.5)[ov6];[ov6]format=yuv420p[vout];[prog_a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "20", + "-profile:v", + "high", + "-maxrate", + "8000k", + "-bufsize", + "16000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "192k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-t", + "5.5", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 5500, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "2.5", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; diff --git a/packages/media/src/clustering.test.ts b/packages/media/src/clustering.test.ts new file mode 100644 index 0000000..c2e333d --- /dev/null +++ b/packages/media/src/clustering.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from "vitest"; +import type { ClusterAsset } from "./clustering.js"; +import { + buildSessions, + calendarDay, + clusterFaces, + clusterThemes, + ruleLabel, +} from "./clustering.js"; + +const at = (iso: string) => new Date(iso); + +function asset( + id: string, + captured: string | null, + extra: Partial = {}, +): ClusterAsset { + return { + id, + kind: "photo", + capturedAt: captured ? at(captured) : null, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + embedding: null, + ...extra, + }; +} + +const vec = (...values: number[]) => new Float32Array(values); + +describe("buildSessions", () => { + it("splits on gaps longer than 45 minutes and sorts by capture time", () => { + const sessions = buildSessions([ + asset("c", "2026-05-01T10:40:00Z"), + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:20:00Z"), + asset("d", "2026-05-01T11:26:00Z"), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]); + expect(sessions[0]!.key).toBe("session:a"); + expect(sessions[0]!.startAt).toEqual(at("2026-05-01T10:00:00Z")); + expect(sessions[0]!.endAt).toEqual(at("2026-05-01T10:40:00Z")); + }); + + it("keeps a gap of exactly 45 minutes in one session", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:45:00Z"), + ]); + expect(sessions).toHaveLength(1); + }); + + it("puts undated assets into one trailing session", () => { + const sessions = buildSessions([ + asset("z", null), + asset("a", "2026-05-01T10:00:00Z"), + asset("m", null), + asset("bad", null, { capturedAt: new Date(Number.NaN) }), + ]); + expect(sessions.map((s) => s.key)).toEqual(["session:a", "session:undated"]); + expect(sessions[1]!.assetIds).toEqual(["bad", "m", "z"]); + expect(sessions[1]!.startAt).toBeNull(); + }); + + it("ignores duplicate ids and mixes photos and videos", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z"), + asset("v", "2026-05-01T10:05:00Z", { kind: "video" }), + asset("a", "2026-05-01T23:00:00Z"), + asset("l", "2026-05-01T10:06:00Z", { kind: "live_photo" }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "v", "l"]]); + }); + + it("splits on a location jump over 2 km and reports a rounded centroid", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z", { gpsLat: 10.0, gpsLon: 20.0 }), + asset("b", "2026-05-01T10:05:00Z"), + asset("c", "2026-05-01T10:10:00Z", { gpsLat: 10.004, gpsLon: 20.004 }), + asset("d", "2026-05-01T10:15:00Z", { gpsLat: 10.1, gpsLon: 20.0 }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]); + expect(sessions[0]!.centroid).toEqual({ lat: 10, lon: 20 }); + expect(sessions[1]!.centroid).toEqual({ lat: 10.1, lon: 20 }); + }); + + it("breaks on a calendar-day change only with a gap over 3 hours, in the given time zone", () => { + const assets = [asset("a", "2026-05-01T22:30:00Z"), asset("b", "2026-05-02T02:00:00Z")]; + // Long gap rule disabled so only the day-break rule decides. + const options = { gapMinutes: 24 * 60 }; + expect(buildSessions(assets, options)).toHaveLength(2); + // In UTC-10 both are on 1 May: one session. + expect(buildSessions(assets, { ...options, timeZone: "Pacific/Honolulu" })).toHaveLength(1); + // Crossing midnight with a short gap stays one session. + expect( + buildSessions( + [asset("x", "2026-05-01T23:50:00Z"), asset("y", "2026-05-02T00:20:00Z")], + options, + ), + ).toHaveLength(1); + expect(calendarDay(at("2026-05-01T22:30:00Z"), "Europe/Berlin")).toBe("2026-05-02"); + }); + + it("merges neighbours with similar embeddings and a gap under 2 hours", () => { + const red = vec(1, 0, 0); + const blue = vec(0, 1, 0); + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z", { embedding: red }), + asset("b", "2026-05-01T11:00:00Z", { embedding: vec(0.95, 0.05, 0) }), + asset("c", "2026-05-01T12:00:00Z", { embedding: blue }), + asset("d", "2026-05-01T15:00:00Z", { embedding: blue }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b"], ["c"], ["d"]]); + }); + + it("is deterministic for any input order", () => { + const assets = [ + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:00:00Z"), + asset("c", "2026-05-02T10:00:00Z"), + asset("d", null), + ]; + const forward = buildSessions(assets); + const reversed = buildSessions([...assets].reverse()); + expect(reversed).toEqual(forward); + expect(forward[0]!.assetIds).toEqual(["a", "b"]); + }); +}); + +describe("clusterThemes and clusterFaces", () => { + const items = [ + { id: "p3", vector: vec(0.9, 0.1, 0) }, + { id: "p1", vector: vec(1, 0, 0) }, + { id: "q1", vector: vec(0, 1, 0) }, + { id: "p2", vector: vec(0.95, 0.05, 0) }, + { id: "q2", vector: vec(0.05, 0.95, 0) }, + { id: "r1", vector: vec(0, 0, 1) }, + ]; + + it("groups by average linkage with stable keys, independent of input order", () => { + const themes = clusterThemes(items); + expect(themes).toEqual([ + { key: "theme:p1", ids: ["p1", "p2", "p3"] }, + { key: "theme:q1", ids: ["q1", "q2"] }, + { key: "theme:r1", ids: ["r1"] }, + ]); + expect(clusterThemes([...items].reverse())).toEqual(themes); + expect(clusterThemes(items, { minSize: 2 }).map((t) => t.key)).toEqual([ + "theme:p1", + "theme:q1", + ]); + expect(clusterThemes(items, { similarity: -1 })).toHaveLength(1); + expect(clusterThemes([])).toEqual([]); + }); + + it("uses average rather than single linkage", () => { + // b is close to a and c, but a and c are far apart: average linkage keeps c out. + const chain = [ + { id: "a", vector: vec(1, 0) }, + { id: "b", vector: vec(Math.cos(0.4), Math.sin(0.4)) }, + { id: "c", vector: vec(Math.cos(1.0), Math.sin(1.0)) }, + ]; + expect(clusterThemes(chain, { similarity: 0.8 }).map((g) => g.ids)).toEqual([ + ["a", "b"], + ["c"], + ]); + }); + + it("returns face groups with a mean centroid", () => { + const faces = clusterFaces([ + { id: "t1", vector: vec(1, 0) }, + { id: "t2", vector: vec(0.8, 0.2) }, + { id: "t3", vector: vec(0, 1) }, + ]); + expect(faces.map((f) => f.key)).toEqual(["person:t1", "person:t3"]); + expect(Array.from(faces[0]!.centroid)).toEqual([0.8999999761581421, 0.10000000149011612]); + expect(() => + clusterFaces([ + { id: "x", vector: vec(1) }, + { id: "y", vector: vec(1, 2) }, + ]), + ).toThrow(); + }); +}); + +describe("ruleLabel", () => { + it("combines the dominant album with a locale-aware date range", () => { + const label = ruleLabel( + { + startAt: at("2026-05-01T10:00:00Z"), + endAt: at("2026-05-03T10:00:00Z"), + albumNames: ["Trip", "Favorites", "Trip"], + }, + "en-US", + ); + expect(label.startsWith("Trip · ")).toBe(true); + expect(label).toMatch(/May 1\D+3, 2026/); + expect( + ruleLabel({ startAt: at("2026-05-01T10:00:00Z"), endAt: null, albumNames: [] }, "de-DE"), + ).toBe("01.05.2026"); + expect(ruleLabel({ startAt: null, endAt: null, albumNames: [" "] })).toBe(""); + expect(ruleLabel({ startAt: null, endAt: null, albumNames: ["B", "A"] })).toBe("A"); + }); +}); diff --git a/packages/media/src/clustering.ts b/packages/media/src/clustering.ts new file mode 100644 index 0000000..31bf572 --- /dev/null +++ b/packages/media/src/clustering.ts @@ -0,0 +1,347 @@ +import type { MediaAssetKind } from "@rakazo/contracts"; +import { cosineSimilarity, haversineKm, roundCoordinate } from "./geo.js"; + +export interface ClusterAsset { + id: string; + kind: MediaAssetKind; + capturedAt: Date | null; + gpsLat: number | null; + gpsLon: number | null; + deviceModel: string | null; + albumNames: string[]; + /** Mean frame embedding; null when not analyzed yet. */ + embedding: Float32Array | null; +} + +export interface SessionOptions { + /** New session after a gap longer than this. Default 45. */ + gapMinutes: number; + /** New session after a location jump larger than this. Default 2. */ + maxDistanceKm: number; + /** New session on calendar-day change when the gap exceeds this. Default 3. */ + dayBreakHours: number; + /** Merge adjacent sessions whose mean embeddings reach this cosine. Default 0.85. */ + mergeSimilarity: number; + /** ...and whose gap is below this. Default 2. */ + mergeMaxGapHours: number; + /** IANA zone for calendar-day boundaries. Default "UTC". */ + timeZone: string; +} + +export const DEFAULT_SESSION_OPTIONS: SessionOptions = { + gapMinutes: 45, + maxDistanceKm: 2, + dayBreakHours: 3, + mergeSimilarity: 0.85, + mergeMaxGapHours: 2, + timeZone: "UTC", +}; + +export interface SessionCluster { + /** Stable across re-clustering: derived from the earliest asset id of the session. */ + key: string; + assetIds: string[]; + startAt: Date | null; + endAt: Date | null; + centroid: { lat: number; lon: number } | null; +} + +const MINUTE_MS = 60_000; +const HOUR_MS = 60 * MINUTE_MS; + +const dayFormatters = new Map(); + +/** Calendar day (YYYY-MM-DD) of an instant in an IANA time zone. */ +export function calendarDay(date: Date, timeZone: string): string { + let formatter = dayFormatters.get(timeZone); + if (!formatter) { + formatter = new Intl.DateTimeFormat("en-CA", { + timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + }); + dayFormatters.set(timeZone, formatter); + } + return formatter.format(date); +} + +type DatedAsset = ClusterAsset & { capturedAt: Date }; +type PositionedAsset = ClusterAsset & { gpsLat: number; gpsLon: number }; + +function hasPosition(asset: ClusterAsset): asset is PositionedAsset { + return ( + asset.gpsLat !== null && + asset.gpsLon !== null && + Number.isFinite(asset.gpsLat) && + Number.isFinite(asset.gpsLon) + ); +} + +function isDated(asset: ClusterAsset): asset is DatedAsset { + return asset.capturedAt !== null && Number.isFinite(asset.capturedAt.getTime()); +} + +/** Mean of the given vectors, or null when none are present or dimensions disagree. */ +export function meanVector(vectors: ArrayLike[]): Float32Array | null { + const first = vectors[0]; + if (!first) return null; + const sum = new Float64Array(first.length); + for (const vector of vectors) { + if (vector.length !== first.length) return null; + for (let i = 0; i < vector.length; i++) sum[i] = sum[i]! + (vector[i] as number); + } + const mean = new Float32Array(first.length); + for (let i = 0; i < sum.length; i++) mean[i] = sum[i]! / vectors.length; + return mean; +} + +function sessionEmbedding(assets: ClusterAsset[]): Float32Array | null { + return meanVector(assets.flatMap((asset) => (asset.embedding ? [asset.embedding] : []))); +} + +function finishSession(assets: ClusterAsset[], key: string): SessionCluster { + const dated = assets.filter(isDated); + const positioned = assets.filter(hasPosition); + const mean = (values: number[]) => values.reduce((sum, v) => sum + v, 0) / values.length; + return { + key, + assetIds: assets.map((asset) => asset.id), + startAt: dated[0]?.capturedAt ?? null, + endAt: dated.at(-1)?.capturedAt ?? null, + centroid: + positioned.length === 0 + ? null + : { + lat: roundCoordinate(mean(positioned.map((a) => a.gpsLat))), + lon: roundCoordinate(mean(positioned.map((a) => a.gpsLon))), + }, + }; +} + +/** + * Deterministic session grouping (spec 4.4). Assets without capturedAt form + * one trailing "undated" session; exact duplicates (same id) are ignored. + * + * A new session starts on a gap longer than gapMinutes, a jump of more than + * maxDistanceKm from the session's last known position, or a calendar-day + * change (in timeZone) with a gap longer than dayBreakHours. Adjacent sessions + * are then merged when their mean embeddings reach mergeSimilarity and the gap + * between them stays below mergeMaxGapHours. Keys derive from the earliest + * asset id, so they survive re-clustering as long as that asset stays first. + */ +export function buildSessions( + assets: ClusterAsset[], + options?: Partial, +): SessionCluster[] { + const opts = { ...DEFAULT_SESSION_OPTIONS, ...options }; + const unique = new Map(); + for (const asset of assets) if (!unique.has(asset.id)) unique.set(asset.id, asset); + + const dated = [...unique.values()] + .filter(isDated) + .sort((a, b) => a.capturedAt.getTime() - b.capturedAt.getTime() || a.id.localeCompare(b.id)); + const undated = [...unique.values()] + .filter((asset) => !isDated(asset)) + .sort((a, b) => a.id.localeCompare(b.id)); + + const split: DatedAsset[][] = []; + let lastPosition: { lat: number; lon: number } | null = null; + for (const asset of dated) { + const current = split.at(-1); + const previous = current?.at(-1); + let startNew = true; + if (current && previous) { + const gap = asset.capturedAt.getTime() - previous.capturedAt.getTime(); + const jumped = + hasPosition(asset) && + lastPosition !== null && + haversineKm(lastPosition, { lat: asset.gpsLat, lon: asset.gpsLon }) > opts.maxDistanceKm; + const dayChanged = + gap > opts.dayBreakHours * HOUR_MS && + calendarDay(asset.capturedAt, opts.timeZone) !== + calendarDay(previous.capturedAt, opts.timeZone); + startNew = gap > opts.gapMinutes * MINUTE_MS || jumped || dayChanged; + } + if (startNew || !current) { + split.push([asset]); + lastPosition = null; + } else { + current.push(asset); + } + if (hasPosition(asset)) lastPosition = { lat: asset.gpsLat, lon: asset.gpsLon }; + } + + // Merge visually similar neighbours that are close in time. + const merged: DatedAsset[][] = []; + for (const session of split) { + const last = merged.at(-1); + if (last) { + const gap = session[0]!.capturedAt.getTime() - last.at(-1)!.capturedAt.getTime(); + const a = sessionEmbedding(last); + const b = sessionEmbedding(session); + if ( + a && + b && + a.length === b.length && + gap < opts.mergeMaxGapHours * HOUR_MS && + cosineSimilarity(a, b) >= opts.mergeSimilarity + ) { + last.push(...session); + continue; + } + } + merged.push([...session]); + } + + const sessions = merged.map((group) => finishSession(group, `session:${group[0]!.id}`)); + if (undated.length > 0) sessions.push(finishSession(undated, "session:undated")); + return sessions; +} + +export interface VectorItem { + id: string; + vector: Float32Array; +} + +/** + * Average-linkage agglomerative clustering cut at a similarity threshold, + * using the nearest-neighbour chain algorithm (O(n²) time and memory). Items + * are sorted by id first and ties resolve to the lower index, so the result + * does not depend on input order. Callers batch very large libraries. + */ +function agglomerate(items: VectorItem[], threshold: number): VectorItem[][] { + const unique = new Map(); + for (const item of items) if (!unique.has(item.id)) unique.set(item.id, item); + const sorted = [...unique.values()].sort((a, b) => a.id.localeCompare(b.id)); + const n = sorted.length; + if (n === 0) return []; + const dim = sorted[0]!.vector.length; + if (sorted.some((item) => item.vector.length !== dim)) throw new Error("vector length mismatch"); + + const sim = new Float64Array(n * n); + for (let i = 0; i < n; i++) { + for (let j = i + 1; j < n; j++) { + const value = cosineSimilarity(sorted[i]!.vector, sorted[j]!.vector); + sim[i * n + j] = value; + sim[j * n + i] = value; + } + } + const size = new Array(n).fill(1); + const members: number[][] = sorted.map((_, i) => [i]); + const active = new Set(members.map((_, i) => i)); + const finished: number[][] = []; + const chain: number[] = []; + + while (active.size > 0) { + if (chain.length === 0) chain.push(Math.min(...active)); + const top = chain.at(-1)!; + const below = chain.length >= 2 ? chain.at(-2)! : -1; + let best = -1; + let bestSim = Number.NEGATIVE_INFINITY; + for (const j of active) { + if (j === top) continue; + const s = sim[top * n + j]!; + if (s > bestSim || (s === bestSim && j < best)) { + best = j; + bestSim = s; + } + } + // Prefer the chain predecessor on ties so the chain cannot cycle. + if (below >= 0 && sim[top * n + below]! === bestSim) best = below; + if (best < 0 || bestSim < threshold) { + // Average linkage is reducible: merges never raise similarity to this cluster again. + active.delete(top); + finished.push(members[top]!); + chain.length = 0; + continue; + } + if (best !== below) { + chain.push(best); + continue; + } + chain.length -= 2; + const a = Math.min(top, below); + const b = Math.max(top, below); + for (const k of active) { + if (k === a || k === b) continue; + const value = + (size[a]! * sim[a * n + k]! + size[b]! * sim[b * n + k]!) / (size[a]! + size[b]!); + sim[a * n + k] = value; + sim[k * n + a] = value; + } + size[a] = size[a]! + size[b]!; + members[a] = [...members[a]!, ...members[b]!]; + active.delete(b); + } + return finished.map((group) => group.sort((x, y) => x - y).map((i) => sorted[i]!)); +} + +function sortGroups(groups: T[]): T[] { + return groups.sort((a, b) => b.ids.length - a.ids.length || a.key.localeCompare(b.key)); +} + +/** + * Agglomerative average-linkage clustering; returns groups with stable keys + * (from the smallest member id), largest first. Default similarity 0.8. + */ +export function clusterThemes( + items: VectorItem[], + options?: { similarity?: number; minSize?: number }, +): Array<{ key: string; ids: string[] }> { + const minSize = options?.minSize ?? 1; + return sortGroups( + agglomerate(items, options?.similarity ?? 0.8) + .filter((group) => group.length >= minSize) + .map((group) => ({ key: `theme:${group[0]!.id}`, ids: group.map((item) => item.id) })), + ); +} + +/** Group face tracks into person suggestions by identity embedding. Local only. Default similarity 0.6. */ +export function clusterFaces( + tracks: VectorItem[], + options?: { similarity?: number }, +): Array<{ key: string; ids: string[]; centroid: Float32Array }> { + return sortGroups( + agglomerate(tracks, options?.similarity ?? 0.6).map((group) => ({ + key: `person:${group[0]!.id}`, + ids: group.map((item) => item.id), + centroid: meanVector(group.map((item) => item.vector))!, + })), + ); +} + +/** + * Rule-based title from the date range and dominant album. Places are not + * named because only rounded coordinates are stored. Returns "" when there is + * nothing to say, so the caller shows its own localized placeholder. + */ +export function ruleLabel( + cluster: { startAt: Date | null; endAt: Date | null; albumNames: string[] }, + locale?: string, + timeZone = "UTC", +): string { + const counts = new Map(); + for (const raw of cluster.albumNames) { + const name = raw.trim(); + if (name) counts.set(name, (counts.get(name) ?? 0) + 1); + } + const album = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]; + + let dates = ""; + const start = cluster.startAt ?? cluster.endAt; + const end = cluster.endAt ?? cluster.startAt; + if (start && end) { + let formatter: Intl.DateTimeFormat; + try { + formatter = new Intl.DateTimeFormat(locale, { dateStyle: "medium", timeZone }); + } catch { + formatter = new Intl.DateTimeFormat("en", { dateStyle: "medium", timeZone }); + } + dates = + calendarDay(start, timeZone) === calendarDay(end, timeZone) + ? formatter.format(start) + : formatter.formatRange(start, end); + } + return [album?.[0], dates].filter(Boolean).join(" · "); +} diff --git a/packages/media/src/compile.ffmpeg.test.ts b/packages/media/src/compile.ffmpeg.test.ts new file mode 100644 index 0000000..cd6051e --- /dev/null +++ b/packages/media/src/compile.ffmpeg.test.ts @@ -0,0 +1,322 @@ +import { execFile, execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import type { RenderPlan } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { CompileInput } from "./compile.js"; +import { compileRenderPlan, escapeFilterText, escapeFilterValue } from "./compile.js"; +import { edlDurationMs } from "./edl.js"; +import { findMetadataLeaks } from "./gate.js"; +import { testEdl, testKit, testProfile } from "./test-fixtures.js"; + +const run = promisify(execFile); + +function onPath(binary: string): boolean { + try { + execFileSync(binary, ["-version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +const hasFfmpeg = onPath("ffmpeg") && onPath("ffprobe"); +const hasRubberband = + hasFfmpeg && + execFileSync("ffmpeg", ["-hide_banner", "-filters"]).toString().includes(" rubberband "); + +async function ffmpeg(args: string[]): Promise { + await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]); +} + +async function execute(plan: RenderPlan): Promise { + for (const step of plan.steps) { + if (step.kind === "write_file") await writeFile(step.path, step.content); + else if (step.kind === "ffmpeg") await run("ffmpeg", ["-loglevel", "error", ...step.argv]); + else throw new Error(`unexpected ${step.kind} step`); + } +} + +interface Probe { + durationMs: number; + width: number; + height: number; + tags: Record; +} + +async function probe(file: string): Promise { + const { stdout } = await run("ffprobe", [ + "-v", + "error", + "-show_format", + "-show_streams", + "-of", + "json", + file, + ]); + const json = JSON.parse(stdout) as { + format: { duration: string; tags?: Record }; + streams: Array<{ + codec_type: string; + width?: number; + height?: number; + tags?: Record; + }>; + }; + const tags: Record = {}; + for (const [key, value] of Object.entries(json.format.tags ?? {})) tags[`format.${key}`] = value; + json.streams.forEach((stream, index) => { + for (const [key, value] of Object.entries(stream.tags ?? {})) + tags[`stream.${index}.${key}`] = value; + }); + const video = json.streams.find((s) => s.codec_type === "video"); + return { + durationMs: Number(json.format.duration) * 1000, + width: video?.width ?? 0, + height: video?.height ?? 0, + tags, + }; +} + +describe.skipIf(!hasFfmpeg)("compiled plans run on ffmpeg", () => { + let dir: string; + let source: string; + let watermark: string; + let music: string; + let intro: string; + let lut: string; + + beforeAll(async () => { + dir = await mkdtemp(path.join(tmpdir(), "media-compile-")); + source = path.join(dir, "source.mp4"); + watermark = path.join(dir, "watermark.png"); + music = path.join(dir, "music.wav"); + intro = path.join(dir, "intro.mov"); + lut = path.join(dir, "brand look.cube"); + // Identity 2x2x2 LUT. + const lattice = []; + for (const b of [0, 1]) + for (const g of [0, 1]) for (const r of [0, 1]) lattice.push(`${r} ${g} ${b}`); + await writeFile(lut, `LUT_3D_SIZE 2\n${lattice.join("\n")}\n`); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc2=size=480x270:rate=30:duration=8", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=8", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + "-shortest", + // Tags a phone would write; the export must drop every one of them. + "-metadata", + "creation_time=2020-01-01T10:00:00Z", + "-metadata", + "location=+00.0000+000.0000/", + "-metadata", + "make=ExampleCam", + source, + ]); + await ffmpeg(["-f", "lavfi", "-i", "color=c=white:s=64x32", "-frames:v", "1", watermark]); + await ffmpeg(["-f", "lavfi", "-i", "sine=frequency=220:duration=6", music]); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc=size=320x180:rate=30:duration=1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + intro, + ]); + }); + + afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }); + }); + + const baseInput = (overrides: Partial): CompileInput => ({ + edl: testEdl(), + profile: testProfile(), + brandKit: testKit(), + quality: "draft", + workDir: dir, + sources: new Map([ + [ + "asset-a", + { + path: source, + width: 480, + height: 270, + durationMs: 8000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + ]), + transcripts: new Map([ + [ + "asset-a", + [ + { startMs: 200, endMs: 600, text: "Wait" }, + { startMs: 650, endMs: 1200, text: "for" }, + { startMs: 1250, endMs: 1800, text: "this!" }, + { startMs: 5100, endMs: 5600, text: "Here" }, + { startMs: 5650, endMs: 6400, text: "it is." }, + ], + ], + ]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.3, cy: 0.5, strength: 1 }, + { tMs: 6000, cx: 0.7, cy: 0.5, strength: 1 }, + ], + ], + ]), + motionClips: new Map(), + files: { + lutPath: null, + watermarkPath: watermark, + fonts: new Map(), + audio: new Map([["music-1", music]]), + }, + videoEncoder: "libx264", + ...overrides, + }); + + it("renders 16:9 to 9:16 with auto-reframe, captions, watermark, ducked music and clean metadata", async () => { + const edl = testEdl({ + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 1500, reframe: "auto" }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: "music-1", + inMs: 0, + outMs: 3500, + atMs: 0, + gainDb: -6, + fadeOutMs: 500, + duckUnderSpeech: true, + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }], + captions: { mode: "both" }, + overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 3500 }], + metadata: { title: "Example teaser" }, + }); + const plan = compileRenderPlan(baseInput({ edl })); + await execute(plan); + const video = plan.outputs.find((o) => o.kind === "video")!.path; + const result = await probe(video); + expect(result.width).toBe(1080); + expect(result.height).toBe(1920); + expect(Math.abs(result.durationMs - edlDurationMs(edl))).toBeLessThanOrEqual(1000 / 30 + 1); + const keys = Object.keys(result.tags).map((k) => k.toLowerCase()); + expect(keys.some((k) => k.includes("creation_time") || k.includes("location"))).toBe(false); + expect(findMetadataLeaks(result.tags)).toEqual([]); + expect(result.tags["format.title"]).toBe("Example teaser"); + + const srt = await readFile(plan.outputs.find((o) => o.kind === "subtitles_srt")!.path, "utf8"); + expect(srt).toContain("Wait for this!"); + const poster = plan.outputs.find((o) => o.kind === "poster")!.path; + expect(existsSync(poster)).toBe(true); + expect((await probe(poster)).tags).toEqual({}); + }); + + it("renders an intro, voice disguise and an escaped text overlay", async () => { + const edl = testEdl({ + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 1000, + outMs: 3000, + atMs: 0, + reframe: "none", + stabilize: true, + color: { exposure: 0.3, temperature: -0.4, saturation: 1.2, useBrandLut: true }, + }, + ], + }, + intro: { templateId: "intro-minimal", durationMs: 1000 }, + overlays: [ + { + id: "t1", + kind: "text", + atMs: 1000, + durationMs: 1500, + text: "50% off: it's [0:v]; rm -rf / %{pts} \\ done", + box: { x: 0.1, y: 0.3, w: 0.8, h: 0.05 }, + }, + { id: "lt", kind: "lower_third", atMs: 1200, durationMs: 1000, text: "Lower third" }, + { id: "pb", kind: "progress_bar", atMs: 1000, durationMs: 2000 }, + ], + redactions: { voice: { method: hasRubberband ? "pitch" : "formant", semitones: 3 } }, + metadata: {}, + }); + const plan = compileRenderPlan( + baseInput({ + edl, + motionClips: new Map([["intro", intro]]), + quality: "final", + files: { ...baseInput({}).files, lutPath: lut }, + }), + ); + await execute(plan); + const result = await probe(plan.outputs[0]!.path); + expect(result.width).toBe(1080); + expect(Math.abs(result.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1); + expect(findMetadataLeaks(result.tags)).toEqual([]); + expect(Object.keys(result.tags).some((k) => k.endsWith("title"))).toBe(false); + }); + + it("draws escaped text exactly like the literal text", async () => { + const text = "it's 50% [0:v]; a,b=c \\ %{pts} 'q'"; + const literal = path.join(dir, "literal.txt"); + await writeFile(literal, text); + const frame = async (filter: string) => { + const { stdout } = await run("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `color=c=black:s=320x80:d=0.1,${filter}`, + "-frames:v", + "1", + "-f", + "framemd5", + "-", + ]); + return stdout.split("\n").filter((line) => line && !line.startsWith("#")); + }; + const escaped = await frame(`drawtext=text=${escapeFilterText(text)}:fontcolor=white:x=4:y=20`); + const reference = await frame( + `drawtext=expansion=none:textfile=${escapeFilterValue(literal)}:fontcolor=white:x=4:y=20`, + ); + const blank = await frame("null"); + expect(escaped).toEqual(reference); + expect(escaped).not.toEqual(blank); + }); +}); diff --git a/packages/media/src/compile.test.ts b/packages/media/src/compile.test.ts new file mode 100644 index 0000000..1a14ebf --- /dev/null +++ b/packages/media/src/compile.test.ts @@ -0,0 +1,576 @@ +import type { RenderPlan } from "@rakazo/contracts"; +import { RenderPlanSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { CaptionWord, CompileInput } from "./compile.js"; +import { + atempoChain, + autoReframe, + buildCaptionCues, + compileRenderPlan, + escapeFilterText, + escapeFilterValue, + outputCaptionWords, + reframeExcluding, + toAss, + toSrt, + toVtt, +} from "./compile.js"; +import { testEdl, testKit, testProfile } from "./test-fixtures.js"; + +const words: CaptionWord[] = [ + { startMs: 200, endMs: 600, text: "Wait" }, + { startMs: 650, endMs: 1200, text: "for" }, + { startMs: 1250, endMs: 1800, text: "this!" }, + { startMs: 5100, endMs: 5600, text: "Here" }, + { startMs: 5650, endMs: 6400, text: "it is --> now." }, +]; + +function input(overrides: Partial = {}): CompileInput { + return { + edl: testEdl(), + profile: testProfile(), + brandKit: testKit(), + quality: "final", + workDir: "/work/render-1", + sources: new Map([ + [ + "asset-a", + { + path: "/media/asset-a/proxy.mp4", + width: 1920, + height: 1080, + durationMs: 10_000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + [ + "asset-b", + { + path: "/media/asset-b/original.mov", + width: 1080, + height: 1920, + durationMs: 8000, + fps: 60, + hasAudio: false, + rotation: 90, + }, + ], + ]), + transcripts: new Map([["asset-a", words]]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.2, cy: 0.5, strength: 1 }, + { tMs: 1500, cx: 0.4, cy: 0.5, strength: 1 }, + { tMs: 5500, cx: 0.8, cy: 0.4, strength: 0.5 }, + ], + ], + ]), + motionClips: new Map(), + files: { + lutPath: "/brand/look's.cube", + watermarkPath: "/brand/watermark.png", + fonts: new Map([["font-1", "/brand/fonts/ExampleSans.ttf"]]), + audio: new Map([ + ["music-1", "/brand/music/bed.wav"], + ["vo-1", "/work/render-1/voiceover.wav"], + ]), + }, + videoEncoder: "libx264", + ...overrides, + }; +} + +function ffmpegStep(plan: RenderPlan, id = "render") { + const step = plan.steps.find((s) => s.id === id); + if (step?.kind !== "ffmpeg") throw new Error(`no ffmpeg step ${id}`); + return step; +} + +function graphOf(plan: RenderPlan): string { + const argv = ffmpegStep(plan).argv; + return argv[argv.indexOf("-filter_complex") + 1]!; +} + +describe("compileRenderPlan", () => { + it("compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + captions: { mode: "both", presetId: "clean" }, + overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 }], + metadata: { title: "Example: teaser" }, + }), + }), + ); + expect(RenderPlanSchema.parse(plan)).toEqual(plan); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("crop=w=608:h=1080:x="); + expect(graph).toContain("scale=1080:1920"); + expect(graph).toContain("loudnorm=I=-14:TP=-1:LRA=11:linear=true"); + expect(graph).toContain("colorchannelmixer=aa=0.6"); + expect(graph).toContain("ass=filename=/work/render-1/captions.ass"); + expect(plan.outputs.map((o) => o.kind)).toEqual([ + "video", + "subtitles_srt", + "subtitles_vtt", + "poster", + ]); + const argv = ffmpegStep(plan).argv; + for (const flag of [ + "-map_metadata", + "-map_chapters", + "+bitexact", + "+faststart", + "yuv420p", + "bt709", + ]) { + expect(argv).toContain(flag); + } + expect(argv).toContain("title=Example: teaser"); + expect(argv.join(" ")).toContain("-c:a aac -b:a 192k -ar 48000"); + }); + + it("compiles transitions, music ducking, voice-over and fades", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" }, + { + id: "c2", + assetId: "asset-a", + inMs: 2000, + outMs: 4000, + atMs: 1500, + reframe: "none", + speed: 0.5, + }, + { + id: "c3", + assetId: "asset-a", + inMs: 3000, + outMs: 4000, + atMs: 6000, + reframe: "none", + }, + { + id: "c4", + assetId: "asset-b", + inMs: 4000, + outMs: 5000, + atMs: 7000, + reframe: "auto", + }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: "music-1", + inMs: 0, + outMs: 8000, + atMs: 0, + gainDb: -8, + fadeInMs: 500, + fadeOutMs: 1000, + duckUnderSpeech: true, + }, + { id: "vo", kind: "voiceover", artifactId: "vo-1", inMs: 0, outMs: 2000, atMs: 1000 }, + ], + }, + transitions: [ + { fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }, + { fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }, + { fromClipId: "c3", toClipId: "c4", kind: "whip", durationMs: 0 }, + ], + targetLufs: -16, + }), + quality: "draft", + }), + ); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("xfade=transition=fade:duration=0.5:offset=1.5"); + expect(graph).toContain("acrossfade=d=0.5"); + expect(graph).toContain("atempo=0.5"); + expect(graph).toContain("color=c=black:s=1080x1920"); // gap from 5.5 s to 6 s + expect(graph).toContain("sidechaincompress"); + expect(graph).toContain("adelay=1000|1000"); + expect(graph).toContain("loudnorm=I=-16"); + expect(ffmpegStep(plan).argv).toEqual(expect.arrayContaining(["veryfast", "28", "3000k"])); + }); + + it("compiles voice disguise, metadata stripping, intro/outro and text overlays", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1500 }, + outro: { templateId: "outro-simple", durationMs: 1000 }, + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 0, + outMs: 3000, + atMs: 0, + reframe: [ + { tMs: 0, cx: 0.3, cy: 0.5, scale: 1 }, + { tMs: 3000, cx: 0.7, cy: 0.5, scale: 1.5 }, + ], + stabilize: true, + color: { exposure: 0.5, temperature: 0.2, saturation: 1.1, useBrandLut: true }, + }, + ], + }, + overlays: [ + { + id: "lt", + kind: "lower_third", + atMs: 2000, + durationMs: 1500, + text: "Ana's: [studio]; 100% real", + }, + { + id: "logo-anim", + kind: "motion", + atMs: 0, + durationMs: 1000, + motion: { templateId: "lower-third-a", durationMs: 1000 }, + }, + { id: "pb", kind: "progress_bar", atMs: 1500, durationMs: 3000 }, + ], + redactions: { voice: { method: "pitch", semitones: -4 } }, + metadata: {}, + }), + motionClips: new Map([["intro", "/work/render-1/intro.mov"]]), + }), + ); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("rubberband=pitch=0.793701:formant=preserved"); + expect(graph).toContain("deshake"); + expect(graph).toContain("lut3d=file=/brand/look\\\\\\'s.cube"); + expect(graph).toContain("concat=n=3:v=1:a=1[prog_v][prog_a]"); + expect(plan.steps.filter((s) => s.kind === "motion").map((s) => s.id)).toEqual([ + "motion-1", + "motion-2", + ]); + const argv = ffmpegStep(plan).argv; + expect(argv.some((a) => a.startsWith("title="))).toBe(false); + expect(argv).toEqual(expect.arrayContaining(["-map_metadata", "-1", "-map_chapters", "-1"])); + }); + + it("uses the asetrate chain for formant disguise and HEVC settings for HEVC profiles", () => { + const plan = compileRenderPlan( + input({ + profile: testProfile({ + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "hevc", + maxBitrateKbps: 6000, + maxFileSizeMb: 100, + }, + export: { poster: false }, + }), + edl: testEdl({ redactions: { voice: { method: "formant", semitones: 12 } }, metadata: {} }), + videoEncoder: "hevc_videotoolbox", + }), + ); + expect(graphOf(plan)).toContain("asetrate=96000,aresample=48000,atempo=0.5"); + const argv = ffmpegStep(plan).argv; + expect(argv).toEqual(expect.arrayContaining(["-tag:v", "hvc1", "-b:v", "4800k"])); + expect(argv).toContain("filter_units=remove_types=39|40"); + expect(plan.outputs.map((o) => o.kind)).toEqual(["video"]); + }); + + it("only ever uses SafeId labels and keeps text out of argv", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + overlays: [ + { id: "t1", kind: "text", atMs: 0, durationMs: 1000, text: "'; rm -rf / [0:v] %{pts}" }, + ], + }), + }), + ); + const graph = graphOf(plan); + for (const label of graph.matchAll(/\[([^\]\\]*)\]/g)) { + expect(label[1]).toMatch(/^[a-z0-9_:-]+$/); + } + expect(graph).toContain("text=\\\\\\'\\; rm -rf / \\[0\\\\:v\\] \\\\\\\\%{pts}"); + expect(ffmpegStep(plan).argv.every((arg) => !arg.includes("\0"))).toBe(true); + }); + + it("refuses unsafe or impossible input instead of guessing", () => { + expect(() => + compileRenderPlan( + input({ + edl: { + ...testEdl(), + overlays: [{ id: "BAD;ID", kind: "text", atMs: 0, durationMs: 1, text: "x" }], + } as never, + }), + ), + ).toThrow(); + expect(() => compileRenderPlan(input({ videoEncoder: "libx265" }))).toThrow( + /does not produce h264/, + ); + expect(() => compileRenderPlan(input({ sources: new Map() }))).toThrow(/no source/); + expect(() => + compileRenderPlan( + input({ + edl: testEdl({ overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 1000 }] }), + files: { ...input().files, watermarkPath: null }, + }), + ), + ).toThrow(/watermark/); + }); + + it("moves very large filtergraphs into a script file", () => { + const video = Array.from({ length: 300 }, (_, i) => ({ + id: `c${i}`, + assetId: "asset-a", + inMs: 0, + outMs: 100, + atMs: i * 100, + reframe: Array.from({ length: 8 }, (_, k) => ({ + tMs: k * 12, + cx: 0.3 + k * 0.05, + cy: 0.5, + scale: 1, + })), + })); + const plan = compileRenderPlan(input({ edl: testEdl({ tracks: { video } }) })); + const script = plan.steps.find((s) => s.id === "filtergraph"); + expect(script?.kind).toBe("write_file"); + expect(ffmpegStep(plan).argv).toContain("-filter_complex_script"); + }); +}); + +describe("escaping", () => { + it("escapes every filtergraph and drawtext special character", () => { + expect(escapeFilterText("plain text")).toBe("plain text"); + expect(escapeFilterText("a:b")).toBe("a\\\\:b"); + expect(escapeFilterText("a,b;c")).toBe("a\\,b\\;c"); + expect(escapeFilterText("[0:v]")).toBe("\\[0\\\\:v\\]"); + expect(escapeFilterText("x=y")).toBe("x\\\\=y"); + expect(escapeFilterText("it's")).toBe("it\\\\\\'s"); + expect(escapeFilterText("100%")).toBe("100\\\\\\\\%"); + expect(escapeFilterText("a\\b")).toBe("a\\\\\\\\\\\\\\\\b"); + expect(escapeFilterText(" line one\r\nline two\u0007 ")).toBe("line one\nline two"); + }); + + /** av_get_token: one parser level; returns the token and what it left unconsumed. */ + function getToken(value: string, terms: string): { token: string; rest: string } { + let token = ""; + let i = 0; + while (i < value.length && !terms.includes(value[i]!)) { + const char = value[i++]!; + if (char === "\\" && i < value.length) token += value[i++]; + else if (char === "'") { + while (i < value.length && value[i] !== "'") token += value[i++]; + i++; + } else token += char; + } + return { token, rest: value.slice(i) }; + } + + function drawtextExpand(value: string): string { + let out = ""; + for (let i = 0; i < value.length; i++) { + if (value[i] === "\\" && i + 1 < value.length) out += value[++i]; + else if (value[i] === "%") throw new Error("expansion sequence reached drawtext"); + else out += value[i]; + } + return out; + } + + it("neutralizes injection attempts", () => { + const attempts = [ + "'; rm -rf /", + "[0:v]", + "%{pts}", + "%{eif:1:d}", + "x:fontfile=/etc/passwd", + "a',drawtext=text='b", + "a\\", + "50% off; [in]null[out]", + ]; + for (const attempt of attempts) { + const graphLevel = getToken(escapeFilterText(attempt), "[],;"); + expect(graphLevel.rest).toBe(""); + const optionLevel = getToken(graphLevel.token, ":"); + expect(optionLevel.rest).toBe(""); + expect(drawtextExpand(optionLevel.token)).toBe(attempt); + } + expect(escapeFilterText("%{pts}")).toBe("\\\\\\\\%{pts}"); + expect(escapeFilterValue("C:\\media\\a,b.cube")).toBe("C\\\\:\\\\\\\\media\\\\\\\\a\\,b.cube"); + expect(() => escapeFilterValue("a\0b")).toThrow(); + }); + + it("chains atempo within its clean range", () => { + expect(atempoChain(1)).toEqual([]); + expect(atempoChain(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]); + expect(atempoChain(3)).toEqual(["atempo=2", "atempo=1.5"]); + expect(atempoChain(4)).toEqual(["atempo=2", "atempo=2"]); + }); +}); + +describe("captions", () => { + it("builds cues by line length, line count, pauses and sentence ends", () => { + const cues = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 1 }); + expect(cues.map((c) => c.text)).toEqual(["Wait for", "this!", "Here", "it is --> now."]); + const two = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 2 }); + expect(two[0]).toMatchObject({ startMs: 200, endMs: 1800, text: "Wait for\nthis!" }); + expect(two[0]!.words).toHaveLength(3); + expect( + buildCaptionCues([{ startMs: 0, endMs: 0, text: " " }], { maxCharsPerLine: 10, maxLines: 2 }), + ).toEqual([]); + const short = buildCaptionCues([{ startMs: 1000, endMs: 1100, text: "Hi" }], { + maxCharsPerLine: 10, + maxLines: 2, + }); + expect(short[0]!.endMs).toBe(1500); + }); + + it("formats SRT and VTT", () => { + const cues = buildCaptionCues(words, { maxCharsPerLine: 32, maxLines: 2 }); + expect(toSrt(cues)).toBe( + "1\n00:00:00,200 --> 00:00:01,800\nWait for this!\n\n2\n00:00:05,100 --> 00:00:06,400\nHere it is -> now.\n", + ); + expect(toVtt(cues)).toBe( + "WEBVTT\n\n00:00:00.200 --> 00:00:01.800\nWait for this!\n\n00:00:05.100 --> 00:00:06.400\nHere it <b>is</b> --> now.\n", + ); + expect(toSrt([{ startMs: 3_723_004, endMs: 3_723_500, text: "a\n\nb", words: [] }])).toBe( + "1\n01:02:03,004 --> 01:02:03,500\na\nb\n", + ); + }); + + it("maps source words to output time across speed and intro", () => { + const edl = testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1000 }, + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 1000, + outMs: 2000, + atMs: 0, + reframe: "none", + speed: 2, + }, + { + id: "c2", + assetId: "asset-a", + inMs: 5000, + outMs: 6000, + atMs: 500, + reframe: "none", + muted: true, + }, + ], + }, + }); + expect(outputCaptionWords(edl, new Map([["asset-a", words]]))).toEqual([ + { startMs: 1125, endMs: 1400, text: "this!" }, + ]); + }); + + it("writes ASS without override injection and with karaoke timing", () => { + const cues = buildCaptionCues( + [ + { startMs: 0, endMs: 400, text: "{\\pos(0,0)}hi" }, + { startMs: 600, endMs: 900, text: "you\\N" }, + ], + { maxCharsPerLine: 32, maxLines: 2 }, + ); + const ass = toAss(cues, { + width: 1080, + height: 1920, + profile: testProfile(), + preset: { + ...testKit().captionPresets[0]!, + template: "karaoke", + highlightColor: "#FFAA00", + uppercase: true, + }, + fallbackTextColor: "#FFFFFF", + position: "lower", + maxCharsPerLine: 32, + wordHighlight: true, + }); + const dialogue = ass.split("\n").find((line) => line.startsWith("Dialogue:"))!; + expect(dialogue).toBe( + "Dialogue: 0,0:00:00.00,0:00:00.90,Default,,0,0,0,,{\\k40}\uFF5B\\\u2060POS(0,0)\uFF5DHI {\\k20}{\\k30}YOU\\\u2060N", + ); + expect(ass).toContain("Style: Default,Example Sans,96,&H0000AAFF,&H00FFFFFF"); + expect(ass).toContain(",2,92,168,422,1"); + }); +}); + +describe("reframing", () => { + const geometry = { + sourceWidth: 1920, + sourceHeight: 1080, + outWidth: 1080, + outHeight: 1920, + durationMs: 4000, + }; + + it("follows the subject smoothly and keeps the crop inside the frame", () => { + const path = autoReframe( + [ + { tMs: 0, cx: 0, cy: 0.5, strength: 1 }, + { tMs: 2000, cx: 0.5, cy: 0.5, strength: 1 }, + { tMs: 4000, cx: 1, cy: 0.5, strength: 1 }, + ], + geometry, + ); + const half = 608 / 1920 / 2; + for (const k of path) { + expect(k.cx).toBeGreaterThanOrEqual(half - 1e-4); + expect(k.cx).toBeLessThanOrEqual(1 - half + 1e-4); + expect(k.cy).toBe(0.5); + expect(k.scale).toBe(1); + } + expect(path[0]!.tMs).toBe(0); + expect(path.at(-1)!.tMs).toBe(4000); + for (let i = 1; i < path.length; i++) + expect(path[i]!.cx).toBeGreaterThanOrEqual(path[i - 1]!.cx); + expect(path.length).toBeLessThanOrEqual(64); + expect(autoReframe([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]); + }); + + it("keeps faces out of the frame or reports that it cannot", () => { + const left = reframeExcluding( + [{ tMs: 100, box: { x: 0.05, y: 0.3, w: 0.15, h: 0.3 } }], + geometry, + ); + expect(left).toHaveLength(1); + const k = left![0]!; + expect(k.tMs).toBe(100); + expect(k.scale).toBe(1); + expect(k.cx - 608 / 1920 / 2).toBeGreaterThanOrEqual(0.22 - 1e-6); + // Faces all over the frame: no crop can exclude them. + const everywhere = [0, 0.25, 0.5, 0.75].map((x) => ({ + tMs: 0, + box: { x, y: 0, w: 0.25, h: 1 }, + })); + expect(reframeExcluding(everywhere, geometry)).toBeNull(); + // A face across the middle forces a tighter crop. + const wide = reframeExcluding([{ tMs: 0, box: { x: 0.2, y: 0.4, w: 0.6, h: 0.2 } }], geometry); + expect(wide![0]!.scale).toBeGreaterThan(1); + expect(reframeExcluding([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]); + }); +}); diff --git a/packages/media/src/compile.ts b/packages/media/src/compile.ts new file mode 100644 index 0000000..ab16f88 --- /dev/null +++ b/packages/media/src/compile.ts @@ -0,0 +1,1369 @@ +import type { + BrandKit, + CaptionStylePreset, + Edl, + EdlAudioClip, + EdlOverlay, + EdlVideoClip, + MediaRenderQuality, + MotionTemplateRef, + NormalizedBox, + PlatformProfile, + ReframeKeyframe, + RenderPlan, + RenderStep, +} from "@rakazo/contracts"; +import { EdlSchema } from "@rakazo/contracts"; +import { + BLENDED_TRANSITIONS, + clipLengthMs, + edlDurationMs, + introMs, + mainProgramMs, + sortedVideoClips, +} from "./edl.js"; + +export interface CompileSource { + /** Redacted intermediate when redactions apply, otherwise original or proxy. */ + path: string; + /** + * Coded size as stored. Rotation 90/270 swaps it for display, because + * ffmpeg auto-rotates while decoding. + */ + width: number; + height: number; + durationMs: number; + fps: number; + hasAudio: boolean; + rotation: 0 | 90 | 180 | 270; + /** A still image (photo). Also assumed when durationMs is 0. */ + still?: boolean; +} + +export interface CaptionWord { + startMs: number; + endMs: number; + text: string; +} + +export interface CompileInput { + edl: Edl; + profile: PlatformProfile; + brandKit: BrandKit | null; + quality: MediaRenderQuality; + /** Absolute work directory for intermediates and outputs. */ + workDir: string; + sources: Map; + /** Source-time words per asset for captions. */ + transcripts: Map; + /** Saliency or subject points per asset for "auto" reframe. */ + subjects: Map>; + /** + * Pre-rendered motion clips keyed by overlay id, "intro", "outro". Missing + * ones become `motion` steps that the renderer runs first. + */ + motionClips: Map; + files: { + lutPath: string | null; + watermarkPath: string | null; + /** Brand-kit font artifact id (or family name) → font file. */ + fonts: Map; + /** artifactId → absolute path for audio clips. */ + audio: Map; + /** First brand-kit logo, for logo overlays without a motion clip. */ + logoPath?: string | null; + }; + /** ffmpeg video encoder name, e.g. libx264, h264_vaapi, h264_videotoolbox. */ + videoEncoder: string; + /** Output time of the poster frame. Defaults to a third into the main program. */ + posterAtMs?: number; +} + +// --------------------------------------------------------------------------- +// Escaping +// --------------------------------------------------------------------------- + +/* + * A filtergraph value passes three parsers: + * 1. the graph parser splits filters on `[ ] , ;` and honours `\` and `'`, + * 2. the option parser splits options on `:` (and `=`), again with `\` and `'`, + * 3. drawtext expands `%{...}` and treats `\x` as a literal `x`. + * Each level is escaped with a backslash, innermost first, so any user text + * arrives verbatim and can never open a new option, filter, or expansion. + */ +const escapeDrawtextLevel = (value: string) => value.replace(/[\\%]/g, "\\$&"); +const escapeOptionLevel = (value: string) => value.replace(/[\\':=]/g, "\\$&"); +const escapeGraphLevel = (value: string) => value.replace(/[\\'[\],;]/g, "\\$&"); + +/** Escape text for ffmpeg drawtext/filtergraph values. */ +export function escapeFilterText(text: string): string { + const normalized = text + .replace(/\r\n?/g, "\n") + // biome-ignore lint/suspicious/noControlCharactersInRegex: strip control characters on purpose + .replace(/[\u0000-\u0009\u000b-\u001f\u007f]/g, "") + .trim(); + return escapeGraphLevel(escapeOptionLevel(escapeDrawtextLevel(normalized))); +} + +/** Escape a file path or expression used as a filter option value. */ +export function escapeFilterValue(value: string): string { + if (value.includes("\0")) throw new Error("filter value contains NUL"); + return escapeGraphLevel(escapeOptionLevel(value)); +} + +// --------------------------------------------------------------------------- +// Small helpers +// --------------------------------------------------------------------------- + +/** Fixed-point number without exponent notation. */ +function num(value: number, digits = 3): string { + const fixed = value.toFixed(digits); + return fixed.includes(".") ? fixed.replace(/\.?0+$/, "") || "0" : fixed; +} + +const sec = (ms: number) => num(ms / 1000); + +function joinPath(dir: string, name: string): string { + return `${dir.replace(/[\\/]+$/, "")}/${name}`; +} + +function dirName(file: string): string { + const index = Math.max(file.lastIndexOf("/"), file.lastIndexOf("\\")); + return index > 0 ? file.slice(0, index) : "."; +} + +function brandColor(hex: string | undefined, fallback: string): string { + return hex ? `0x${hex.slice(1).toUpperCase()}` : fallback; +} + +function even(value: number): number { + return Math.max(2, Math.floor(value / 2) * 2); +} + +const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value)); + +function enableBetween(startMs: number, endMs: number): string { + return escapeFilterValue(`between(t,${sec(startMs)},${sec(endMs)})`); +} + +/** atempo accepts 0.5..2 per instance for clean results; chain for the rest. */ +export function atempoChain(speed: number): string[] { + const factors: number[] = []; + let rest = speed; + while (rest > 2) { + factors.push(2); + rest /= 2; + } + while (rest < 0.5) { + factors.push(0.5); + rest /= 0.5; + } + if (Math.abs(rest - 1) > 1e-9 || factors.length === 0) factors.push(rest); + return factors.filter((f) => Math.abs(f - 1) > 1e-9).map((f) => `atempo=${num(f, 6)}`); +} + +// --------------------------------------------------------------------------- +// Reframing +// --------------------------------------------------------------------------- + +interface FrameGeometry { + sourceWidth: number; + sourceHeight: number; + outWidth: number; + outHeight: number; + durationMs: number; +} + +/** Largest crop with the output aspect inside the source, shrunk by scale. */ +function cropSize(geometry: FrameGeometry, scale: number): { w: number; h: number } { + const outAspect = geometry.outWidth / geometry.outHeight; + let w: number; + let h: number; + if (geometry.sourceWidth / geometry.sourceHeight > outAspect) { + h = geometry.sourceHeight; + w = h * outAspect; + } else { + w = geometry.sourceWidth; + h = w / outAspect; + } + const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2); + return { + w: Math.min(geometry.sourceWidth, roundEven(w / scale)), + h: Math.min(geometry.sourceHeight, roundEven(h / scale)), + }; +} + +const AUTO_STEP_MS = 500; +const AUTO_WINDOW_MS = 750; +const AUTO_SMOOTHING = 0.35; +const AUTO_TOLERANCE = 0.01; +const MAX_KEYFRAMES = 64; + +function smooth(values: number[]): number[] { + const forward = [...values]; + for (let i = 1; i < forward.length; i++) { + forward[i] = forward[i - 1]! + AUTO_SMOOTHING * (forward[i]! - forward[i - 1]!); + } + const backward = [...forward]; + for (let i = backward.length - 2; i >= 0; i--) { + backward[i] = backward[i + 1]! + AUTO_SMOOTHING * (backward[i]! - backward[i + 1]!); + } + return backward; +} + +/** Keep only keyframes that linear interpolation cannot reproduce, at most MAX_KEYFRAMES. */ +function simplify(keyframes: ReframeKeyframe[]): ReframeKeyframe[] { + if (keyframes.length <= 2) return keyframes; + const kept: ReframeKeyframe[] = [keyframes[0]!]; + for (let i = 1; i < keyframes.length - 1; i++) { + const last = kept.at(-1)!; + const next = keyframes[i + 1]!; + const point = keyframes[i]!; + const f = (point.tMs - last.tMs) / (next.tMs - last.tMs); + const cx = last.cx + (next.cx - last.cx) * f; + const cy = last.cy + (next.cy - last.cy) * f; + if (Math.abs(cx - point.cx) > AUTO_TOLERANCE || Math.abs(cy - point.cy) > AUTO_TOLERANCE) { + kept.push(point); + } + } + kept.push(keyframes.at(-1)!); + if (kept.length <= MAX_KEYFRAMES) return kept; + const step = (kept.length - 1) / (MAX_KEYFRAMES - 1); + return Array.from({ length: MAX_KEYFRAMES }, (_, i) => kept[Math.round(i * step)]!); +} + +/** + * Smoothed crop path that keeps the subject in frame for the output aspect. + * Subjects are weighted by strength in a ±750 ms window every 500 ms, the + * path is smoothed forwards and backwards (no lag), clamped so the crop stays + * inside the source, and reduced to the keyframes that matter. Times are the + * subjects' time base, from 0 to durationMs. + */ +export function autoReframe( + subjects: Array<{ tMs: number; cx: number; cy: number; strength: number }>, + input: FrameGeometry, +): ReframeKeyframe[] { + const crop = cropSize(input, 1); + const halfW = crop.w / input.sourceWidth / 2; + const halfH = crop.h / input.sourceHeight / 2; + const clampX = (x: number) => clamp(x, halfW, 1 - halfW); + const clampY = (y: number) => clamp(y, halfH, 1 - halfH); + const usable = subjects + .filter((s) => s.strength > 0 && Number.isFinite(s.cx) && Number.isFinite(s.cy)) + .sort((a, b) => a.tMs - b.tMs); + if (usable.length === 0) return [{ tMs: 0, cx: clampX(0.5), cy: clampY(0.5), scale: 1 }]; + + const times: number[] = []; + for (let t = 0; t < input.durationMs; t += AUTO_STEP_MS) times.push(t); + times.push(Math.max(0, input.durationMs)); + const xs: number[] = []; + const ys: number[] = []; + for (const t of times) { + let weight = 0; + let x = 0; + let y = 0; + for (const subject of usable) { + const distance = Math.abs(subject.tMs - t); + if (distance > AUTO_WINDOW_MS) continue; + const w = subject.strength * (1 - distance / (AUTO_WINDOW_MS * 2)); + weight += w; + x += subject.cx * w; + y += subject.cy * w; + } + if (weight > 0) { + xs.push(x / weight); + ys.push(y / weight); + } else if (xs.length > 0) { + xs.push(xs.at(-1)!); + ys.push(ys.at(-1)!); + } else { + const nearest = usable.reduce((best, s) => + Math.abs(s.tMs - t) < Math.abs(best.tMs - t) ? s : best, + ); + xs.push(nearest.cx); + ys.push(nearest.cy); + } + } + const sx = smooth(xs); + const sy = smooth(ys); + const round = (v: number) => Math.round(v * 10_000) / 10_000; + return simplify( + times.map((tMs, i) => ({ + tMs: Math.round(tMs), + cx: round(clampX(sx[i]!)), + cy: round(clampY(sy[i]!)), + scale: 1, + })), + ); +} + +const REFRAME_SCALES = [1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4]; +const FACE_MARGIN = 0.02; + +/** + * Crop path that keeps every given face box outside the output frame, or + * null if impossible. The window is static for the whole shot (a moving + * window could sweep across a face between keyframes): it avoids the union of + * all face boxes plus a 2 % margin, prefers the widest crop, and then the + * position closest to the frame centre. + */ +export function reframeExcluding( + faces: Array<{ tMs: number; box: NormalizedBox }>, + input: FrameGeometry, +): ReframeKeyframe[] | null { + const tMs = faces.reduce((min, face) => Math.min(min, face.tMs), Number.POSITIVE_INFINITY); + const boxes = faces.map(({ box }) => ({ + x0: clamp(box.x - FACE_MARGIN, 0, 1), + y0: clamp(box.y - FACE_MARGIN, 0, 1), + x1: clamp(box.x + box.w + FACE_MARGIN, 0, 1), + y1: clamp(box.y + box.h + FACE_MARGIN, 0, 1), + })); + for (const scale of REFRAME_SCALES) { + const crop = cropSize(input, scale); + const w = crop.w / input.sourceWidth; + const h = crop.h / input.sourceHeight; + const xs = new Set([0.5, w / 2, 1 - w / 2]); + const ys = new Set([0.5, h / 2, 1 - h / 2]); + for (const b of boxes) { + xs.add(b.x0 - w / 2); + xs.add(b.x1 + w / 2); + ys.add(b.y0 - h / 2); + ys.add(b.y1 + h / 2); + } + const candidates: Array<{ cx: number; cy: number; distance: number }> = []; + for (const rawX of xs) { + for (const rawY of ys) { + const cx = clamp(rawX, w / 2, 1 - w / 2); + const cy = clamp(rawY, h / 2, 1 - h / 2); + const clear = boxes.every( + (b) => + cx + w / 2 <= b.x0 + 1e-9 || + cx - w / 2 >= b.x1 - 1e-9 || + cy + h / 2 <= b.y0 + 1e-9 || + cy - h / 2 >= b.y1 - 1e-9, + ); + if (clear) candidates.push({ cx, cy, distance: Math.hypot(cx - 0.5, cy - 0.5) }); + } + } + candidates.sort((a, b) => a.distance - b.distance || a.cx - b.cx || a.cy - b.cy); + const best = candidates[0]; + if (best) { + const round = (v: number) => Math.round(v * 10_000) / 10_000; + return [ + { + tMs: Number.isFinite(tMs) ? Math.round(tMs) : 0, + cx: round(best.cx), + cy: round(best.cy), + scale, + }, + ]; + } + } + return null; +} + +/** Piecewise-linear expression of `t` (seconds) through the given points. */ +function piecewise(points: Array<{ t: number; v: number }>): string { + if (points.length === 1) return num(points[0]!.v, 2); + let expression = num(points.at(-1)!.v, 2); + for (let i = points.length - 2; i >= 0; i--) { + const a = points[i]!; + const b = points[i + 1]!; + const span = Math.max(b.t - a.t, 0.001); + const segment = `${num(a.v, 2)}+(${num(b.v - a.v, 2)})*(t-${num(a.t)})/${num(span)}`; + expression = `if(lt(t,${num(b.t)}),${segment},${expression})`; + } + return `if(lt(t,${num(points[0]!.t)}),${num(points[0]!.v, 2)},${expression})`; +} + +/** crop+scale filters for keyframes in source time; uses the tightest scale of the path. */ +function cropFilters( + keyframes: ReframeKeyframe[], + clip: EdlVideoClip, + geometry: FrameGeometry, +): string[] { + const scale = Math.max(...keyframes.map((k) => k.scale)); + const crop = cropSize(geometry, scale); + const points = [...keyframes] + .sort((a, b) => a.tMs - b.tMs) + .map((k) => ({ + t: Math.max(0, (k.tMs - clip.inMs) / 1000), + x: clamp(k.cx * geometry.sourceWidth - crop.w / 2, 0, geometry.sourceWidth - crop.w), + y: clamp(k.cy * geometry.sourceHeight - crop.h / 2, 0, geometry.sourceHeight - crop.h), + })); + const x = piecewise(points.map((p) => ({ t: p.t, v: p.x }))); + const y = piecewise(points.map((p) => ({ t: p.t, v: p.y }))); + return [ + `crop=w=${crop.w}:h=${crop.h}:x=${escapeFilterValue(x)}:y=${escapeFilterValue(y)}`, + `scale=${geometry.outWidth}:${geometry.outHeight}`, + ]; +} + +function fitFilters(width: number, height: number): string[] { + return [ + `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`, + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, + ]; +} + +// --------------------------------------------------------------------------- +// Captions +// --------------------------------------------------------------------------- + +export interface CaptionCue { + startMs: number; + endMs: number; + text: string; + /** Word timings inside the cue for karaoke highlight. */ + words: CaptionWord[]; +} + +const CUE_PAUSE_MS = 1000; +const CUE_MAX_MS = 7000; +const CUE_MIN_MS = 500; + +function cleanWord(word: CaptionWord): CaptionWord | null { + const text = word.text.replace(/\s+/g, " ").trim(); + if (!text || !(word.endMs >= word.startMs)) return null; + return { startMs: word.startMs, endMs: word.endMs, text }; +} + +/** Greedy line wrap; a single word longer than the limit gets its own line. */ +function wrapWords(words: CaptionWord[], maxChars: number): CaptionWord[][] { + const lines: CaptionWord[][] = []; + let line: CaptionWord[] = []; + let length = 0; + for (const word of words) { + const extra = line.length === 0 ? word.text.length : length + 1 + word.text.length; + if (line.length > 0 && extra > maxChars) { + lines.push(line); + line = [word]; + length = word.text.length; + } else { + line.push(word); + length = extra; + } + } + if (line.length > 0) lines.push(line); + return lines; +} + +/** + * Group words into cues of at most maxLines lines of maxCharsPerLine. A cue + * also ends at a pause over 1 s, after sentence punctuation, or at 7 s. Each + * cue is shown at least 500 ms unless the next one starts earlier. + */ +export function buildCaptionCues( + words: CaptionWord[], + options: { maxCharsPerLine: number; maxLines: number }, +): CaptionCue[] { + const clean = words + .map(cleanWord) + .filter((w): w is CaptionWord => w !== null) + .sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs); + const groups: CaptionWord[][] = []; + let current: CaptionWord[] = []; + for (const word of clean) { + const last = current.at(-1); + const first = current[0]; + const breaks = + last !== undefined && + first !== undefined && + (word.startMs - last.endMs > CUE_PAUSE_MS || + /[.!?…]["»”)]?$/.test(last.text) || + word.endMs - first.startMs > CUE_MAX_MS || + wrapWords([...current, word], options.maxCharsPerLine).length > options.maxLines); + if (breaks) { + groups.push(current); + current = []; + } + current.push(word); + } + if (current.length > 0) groups.push(current); + + return groups.map((group, index) => { + const startMs = group[0]!.startMs; + const next = groups[index + 1]?.[0]?.startMs ?? Number.POSITIVE_INFINITY; + const spoken = Math.max(...group.map((w) => w.endMs)); + const endMs = Math.max(spoken, Math.min(startMs + CUE_MIN_MS, next)); + return { + startMs, + endMs: Math.max(endMs, startMs + 1), + text: wrapWords(group, options.maxCharsPerLine) + .map((line) => line.map((w) => w.text).join(" ")) + .join("\n"), + words: group, + }; + }); +} + +function clock(ms: number, separator: "," | "."): string { + const total = Math.max(0, Math.round(ms)); + const hours = Math.floor(total / 3_600_000); + const minutes = Math.floor((total % 3_600_000) / 60_000); + const seconds = Math.floor((total % 60_000) / 1000); + const millis = total % 1000; + const pad = (v: number, n = 2) => String(v).padStart(n, "0"); + return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}${separator}${pad(millis, 3)}`; +} + +/** Cue text without blank lines, which would end a cue early in both formats. */ +function cueLines(text: string): string { + return text + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .join("\n"); +} + +export function toSrt(cues: CaptionCue[]): string { + return cues + .map( + (cue, index) => + `${index + 1}\n${clock(cue.startMs, ",")} --> ${clock(cue.endMs, ",")}\n${cueLines( + cue.text, + ).replace(/-->/g, "->")}\n`, + ) + .join("\n"); +} + +export function toVtt(cues: CaptionCue[]): string { + const escapeCue = (text: string) => + text.replace(/&/g, "&").replace(//g, ">"); + return `WEBVTT\n\n${cues + .map( + (cue) => + `${clock(cue.startMs, ".")} --> ${clock(cue.endMs, ".")}\n${escapeCue(cueLines(cue.text))}\n`, + ) + .join("\n")}`; +} + +/** ASS colour &HAABBGGRR from #RRGGBB. */ +function assColor(hex: string, alpha = 0): string { + const r = hex.slice(1, 3); + const g = hex.slice(3, 5); + const b = hex.slice(5, 7); + return `&H${alpha.toString(16).padStart(2, "0")}${b}${g}${r}`.toUpperCase(); +} + +function assTime(ms: number): string { + const total = Math.max(0, Math.round(ms / 10)); + const hours = Math.floor(total / 360_000); + const minutes = Math.floor((total % 360_000) / 6000); + const seconds = Math.floor((total % 6000) / 100); + const centis = total % 100; + const pad = (v: number) => String(v).padStart(2, "0"); + return `${hours}:${pad(minutes)}:${pad(seconds)}.${pad(centis)}`; +} + +/** Plain text for an ASS event: no override blocks, no escape sequences. */ +function assText(text: string): string { + return text.replace(/\\/g, "\\⁠").replace(/\{/g, "｛").replace(/\}/g, "｝"); +} + +export function toAss( + cues: CaptionCue[], + options: { + width: number; + height: number; + profile: PlatformProfile; + preset: CaptionStylePreset | null; + fallbackTextColor: string; + position: "lower" | "middle" | "upper"; + maxCharsPerLine: number; + wordHighlight: boolean; + language?: string; + }, +): string { + const { width, height, profile, preset } = options; + const zones = profile.safeZones; + const margin = Math.round(height * 0.02); + const alignment = { lower: 2, middle: 5, upper: 8 }[options.position]; + const marginV = + options.position === "upper" + ? Math.round((zones.topPct / 100) * height) + margin + : options.position === "lower" + ? Math.round((zones.bottomPct / 100) * height) + margin + : 0; + const marginL = Math.round((zones.leftPct / 100) * width) + margin; + const marginR = Math.round((zones.rightPct / 100) * width) + margin; + const textColor = preset?.textColor ?? options.fallbackTextColor; + const highlight = preset?.highlightColor ?? textColor; + const karaoke = options.wordHighlight || preset?.template === "karaoke"; + const bold = preset?.template === "bold_social"; + const fontName = (preset?.fontFamily ?? "Sans").replace(/[,\r\n]/g, " "); + const fontSize = Math.round(((preset?.sizePct ?? 5) / 100) * height); + const background = preset?.backgroundColor; + const style = [ + "Default", + fontName, + fontSize, + assColor(karaoke ? highlight : textColor), + assColor(textColor), + background ? assColor(background) : "&H00000000", + background ? assColor(background, 0x40) : "&H80000000", + bold ? -1 : 0, + 0, + 0, + 0, + 100, + 100, + 0, + 0, + background ? 3 : 1, + bold ? 4 : 2, + 0, + alignment, + marginL, + marginR, + marginV, + 1, + ].join(","); + const transform = (text: string) => + preset?.uppercase ? text.toLocaleUpperCase(options.language) : text; + const events = cues.map((cue) => { + let body: string; + if (karaoke && cue.words.length > 0) { + let cursor = cue.startMs; + body = wrapWords(cue.words, options.maxCharsPerLine) + .map((line) => + line + .map((word) => { + const gap = Math.max(0, Math.round((word.startMs - cursor) / 10)); + const length = Math.max(1, Math.round((word.endMs - word.startMs) / 10)); + cursor = Math.max(cursor, word.endMs); + return `${gap > 0 ? `{\\k${gap}}` : ""}{\\k${length}}${assText(transform(word.text))}`; + }) + .join(" "), + ) + .join("\\N"); + } else { + body = cueLines(transform(cue.text)).split("\n").map(assText).join("\\N"); + } + return `Dialogue: 0,${assTime(cue.startMs)},${assTime(cue.endMs)},Default,,0,0,0,,${body}`; + }); + return [ + "[Script Info]", + "ScriptType: v4.00+", + `PlayResX: ${width}`, + `PlayResY: ${height}`, + "WrapStyle: 2", + "ScaledBorderAndShadow: yes", + "", + "[V4+ Styles]", + "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding", + `Style: ${style}`, + "", + "[Events]", + "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text", + ...events, + "", + ].join("\n"); +} + +/** Caption words on the output timeline, from each unmuted clip's transcript. */ +export function outputCaptionWords( + edl: Edl, + transcripts: Map, +): CaptionWord[] { + const offset = introMs(edl); + const words: CaptionWord[] = []; + for (const clip of sortedVideoClips(edl)) { + if (clip.muted) continue; + for (const word of transcripts.get(clip.assetId) ?? []) { + const start = Math.max(word.startMs, clip.inMs); + const end = Math.min(word.endMs, clip.outMs); + if (word.startMs < clip.inMs || start >= clip.outMs || end < start) continue; + words.push({ + startMs: Math.round(offset + clip.atMs + (start - clip.inMs) / clip.speed), + endMs: Math.round(offset + clip.atMs + (end - clip.inMs) / clip.speed), + text: word.text, + }); + } + } + return words; +} + +// --------------------------------------------------------------------------- +// Compiler +// --------------------------------------------------------------------------- + +const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo"; +const XFADE_KIND = { crossfade: "fade", dip: "fadeblack", whip: "slideleft" } as const; +const MAX_INLINE_GRAPH = 60_000; + +function encoderFamily(encoder: string): "h264" | "hevc" | null { + if (/^(libx264|h264_)/.test(encoder)) return "h264"; + if (/^(libx265|hevc_)/.test(encoder)) return "hevc"; + return null; +} + +function encodingArgs(input: CompileInput): string[] { + const { profile, quality, videoEncoder } = input; + const family = encoderFamily(videoEncoder); + if (family !== profile.video.codec) { + throw new Error(`encoder ${videoEncoder} does not produce ${profile.video.codec}`); + } + const draft = quality === "draft"; + const maxKbps = draft + ? Math.min(profile.video.maxBitrateKbps, 3000) + : profile.video.maxBitrateKbps; + const args = ["-c:v", videoEncoder]; + if (videoEncoder === "libx264" || videoEncoder === "libx265") { + args.push("-preset", draft ? "veryfast" : "medium", "-crf", draft ? "28" : "20"); + if (videoEncoder === "libx264") args.push("-profile:v", "high"); + } else { + args.push("-b:v", `${Math.round(maxKbps * 0.8)}k`); + } + args.push("-maxrate", `${maxKbps}k`, "-bufsize", `${maxKbps * 2}k`); + if (!videoEncoder.endsWith("_vaapi")) args.push("-pix_fmt", "yuv420p"); + args.push( + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + ); + // Drop SEI messages, which carry encoder name, version and settings. + args.push( + "-bsf:v", + family === "h264" ? "filter_units=remove_types=6" : "filter_units=remove_types=39|40", + ); + if (family === "hevc") args.push("-tag:v", "hvc1"); + args.push("-r", num(input.edl.output.fps, 3)); + args.push("-c:a", "aac", "-b:a", draft ? "128k" : "192k", "-ar", "48000", "-ac", "2"); + return args; +} + +/** Strip every inherited tag, chapter, and encoder string; only the plan's title is written. */ +function metadataArgs(title: string | undefined): string[] { + const args = [ + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + ]; + const clean = title?.replace(/[\0\r\n]+/g, " ").trim(); + if (clean) args.push("-metadata", `title=${clean}`); + return args; +} + +class Inputs { + readonly args: string[] = []; + private count = 0; + add(...args: string[]): number { + if (args.some((arg) => arg.includes("\0"))) throw new Error("argument contains NUL"); + this.args.push(...args); + return this.count++; + } +} + +function captionPreset(edl: Edl, kit: BrandKit | null): CaptionStylePreset | null { + const presets = kit?.captionPresets ?? []; + return presets.find((p) => p.id === edl.captions?.presetId) ?? presets[0] ?? null; +} + +function pickFont(kit: BrandKit | null, fonts: Map, role: string): string | null { + const preferred = kit?.fonts.find((font) => font.role === role) ?? kit?.fonts[0]; + if (preferred) { + const path = fonts.get(preferred.artifactId) ?? fonts.get(preferred.family); + if (path) return path; + } + return fonts.values().next().value ?? null; +} + +/** + * Pure compiler from a validated EDL to ffmpeg argv steps plus caption files. + * No string ever reaches a shell; text is escaped for filtergraph and ASS, + * every label comes from SafeId values. + * + * Loudness: a single loudnorm pass with linear=true targets the profile (or + * plan) loudness and true peak. Without measured values loudnorm runs in its + * dynamic mode; a renderer may run a measurement pass first and append the + * measured_* options to that filter. + * + * Face, mark, and region redactions are applied before compilation: sources + * already point at redacted intermediates. + */ +export function compileRenderPlan(input: CompileInput): RenderPlan { + const edl = EdlSchema.parse(input.edl); + const { profile, brandKit: kit, workDir } = input; + const W = edl.output.width; + const H = edl.output.height; + const F = edl.output.fps; + const totalMs = edlDurationMs(edl); + const intro = introMs(edl); + const mainMs = mainProgramMs(edl); + const steps: RenderStep[] = []; + const outputs: RenderPlan["outputs"] = []; + const inputs = new Inputs(); + const graph: string[] = []; + let motionCount = 0; + + const motionInput = (key: string, template: MotionTemplateRef, width: number, height: number) => { + let path = input.motionClips.get(key); + if (!path) { + motionCount += 1; + path = joinPath(workDir, `motion-${motionCount}.mov`); + steps.push({ + kind: "motion", + id: `motion-${motionCount}`, + template, + width, + height, + fps: F, + outPath: path, + }); + } + return inputs.add("-i", path); + }; + + const finishVideo = (lengthMs: number) => [ + `fps=${num(F, 3)}`, + "format=yuv420p", + "tpad=stop_mode=clone:stop_duration=2", + `trim=duration=${sec(lengthMs)}`, + "setpts=PTS-STARTPTS", + ]; + const finishAudio = (lengthMs: number) => [ + AUDIO_FORMAT, + "apad", + `atrim=duration=${sec(lengthMs)}`, + "asetpts=PTS-STARTPTS", + ]; + const silence = (label: string, lengthMs: number) => + graph.push(`anullsrc=r=48000:cl=stereo,${finishAudio(lengthMs).join(",")}[${label}]`); + + // Per-clip chains. + interface Part { + v: string; + a: string; + lengthMs: number; + } + const clipParts = new Map(); + for (const clip of sortedVideoClips(edl)) { + const source = input.sources.get(clip.assetId); + if (!source) throw new Error(`no source for asset in clip ${clip.id}`); + const still = source.still === true || source.durationMs === 0; + const sourceLength = clip.outMs - clip.inMs; + const lengthMs = clipLengthMs(clip); + const index = still + ? inputs.add( + "-loop", + "1", + "-framerate", + num(F, 3), + "-t", + sec(sourceLength), + "-i", + source.path, + ) + : inputs.add("-ss", sec(clip.inMs), "-t", sec(sourceLength), "-i", source.path); + const rotated = source.rotation === 90 || source.rotation === 270; + const geometry: FrameGeometry = { + sourceWidth: rotated ? source.height : source.width, + sourceHeight: rotated ? source.width : source.height, + outWidth: W, + outHeight: H, + durationMs: sourceLength, + }; + const video: string[] = []; + if (clip.stabilize && !still) video.push("deshake"); + if (clip.reframe === "none") { + video.push(...fitFilters(W, H)); + } else { + const keyframes = + clip.reframe === "auto" + ? autoReframe( + (input.subjects.get(clip.assetId) ?? []) + .filter( + (s) => + s.tMs >= clip.inMs - AUTO_WINDOW_MS && s.tMs <= clip.outMs + AUTO_WINDOW_MS, + ) + .map((s) => ({ ...s, tMs: s.tMs - clip.inMs })), + geometry, + ).map((k) => ({ ...k, tMs: k.tMs + clip.inMs })) + : clip.reframe; + video.push(...cropFilters(keyframes, clip, geometry)); + } + video.push("setsar=1", `setpts=(PTS-STARTPTS)/${num(clip.speed, 6)}`); + const color = clip.color; + if (color) { + const brightness = clamp(color.exposure * 0.1, -1, 1); + if (brightness !== 0 || color.contrast !== 1 || color.saturation !== 1) { + video.push( + `eq=brightness=${num(brightness)}:contrast=${num(color.contrast)}:saturation=${num(color.saturation)}`, + ); + } + if (color.temperature !== 0) { + const shift = num(color.temperature * 0.1); + const inverse = num(-color.temperature * 0.1); + video.push(`colorbalance=rm=${shift}:bm=${inverse}`); + } + if (color.useBrandLut) { + if (!input.files.lutPath) throw new Error(`clip ${clip.id} needs the brand LUT file`); + video.push(`lut3d=file=${escapeFilterValue(input.files.lutPath)}`); + } + } + video.push(...finishVideo(lengthMs)); + graph.push(`[${index}:v]${video.join(",")}[v_${clip.id}]`); + + if (source.hasAudio && !clip.muted && !still) { + const audio = ["asetpts=PTS-STARTPTS", ...atempoChain(clip.speed), "aresample=48000"]; + graph.push(`[${index}:a]${[...audio, ...finishAudio(lengthMs)].join(",")}[a_${clip.id}]`); + } else { + silence(`a_${clip.id}`, lengthMs); + } + clipParts.set(clip.id, { v: `v_${clip.id}`, a: `a_${clip.id}`, lengthMs }); + } + + // Main program: concatenate runs, blend where transitions ask for it, fill gaps with black. + const blends = new Map( + edl.transitions + .filter((t) => BLENDED_TRANSITIONS.has(t.kind) && t.durationMs > 0) + .map((t) => [`${t.fromClipId}>${t.toClipId}`, t] as const), + ); + let mixCount = 0; + const nextLabel = (prefix: string) => `${prefix}${++mixCount}`; + let accumulated: Part | null = null; + let queue: Part[] = []; + const flush = () => { + const parts = accumulated ? [accumulated, ...queue] : queue; + queue = []; + if (parts.length <= 1) { + accumulated = parts[0] ?? accumulated; + return; + } + const v = nextLabel("cv"); + const a = nextLabel("ca"); + graph.push( + `${parts.map((p) => `[${p.v}][${p.a}]`).join("")}concat=n=${parts.length}:v=1:a=1[${v}][${a}]`, + ); + accumulated = { v, a, lengthMs: parts.reduce((sum, p) => sum + p.lengthMs, 0) }; + }; + let cursor = 0; + let previous: EdlVideoClip | null = null; + for (const clip of sortedVideoClips(edl)) { + const part = clipParts.get(clip.id)!; + const blend = previous ? blends.get(`${previous.id}>${clip.id}`) : undefined; + if (blend) { + flush(); + const base = accumulated as Part | null; + if (!base) throw new Error("transition without a preceding clip"); + const v = nextLabel("xv"); + const a = nextLabel("xa"); + const kind = XFADE_KIND[blend.kind as keyof typeof XFADE_KIND]; + graph.push( + `[${base.v}][${part.v}]xfade=transition=${kind}:duration=${sec(blend.durationMs)}:offset=${sec( + base.lengthMs - blend.durationMs, + )}[${v}]`, + `[${base.a}][${part.a}]acrossfade=d=${sec(blend.durationMs)}:c1=tri:c2=tri[${a}]`, + ); + accumulated = { v, a, lengthMs: base.lengthMs + part.lengthMs - blend.durationMs }; + } else { + const gap = clip.atMs - cursor; + if (gap > 1) { + const v = nextLabel("gv"); + const a = nextLabel("ga"); + graph.push(`color=c=black:s=${W}x${H}:r=${num(F, 3)},${finishVideo(gap).join(",")}[${v}]`); + silence(a, gap); + queue.push({ v, a, lengthMs: gap }); + } + queue.push(part); + } + cursor = Math.max(cursor, clip.atMs + part.lengthMs); + previous = clip; + } + flush(); + const main = accumulated as Part | null; + if (!main) throw new Error("plan has no video clips"); + + let mainAudio = main.a; + const voice = edl.redactions.voice; + if (voice) { + const ratio = 2 ** (voice.semitones / 12); + const label = nextLabel("vd"); + const filter = + voice.method === "pitch" + ? `rubberband=pitch=${num(ratio, 6)}:formant=preserved` + : `asetrate=${num(48000 * ratio, 2)},aresample=48000,atempo=${num(1 / ratio, 6)}`; + graph.push(`[${mainAudio}]${filter},${finishAudio(main.lengthMs).join(",")}[${label}]`); + mainAudio = label; + } + + // Intro and outro. + const program: Part[] = []; + const bookend = (key: "intro" | "outro", ref: MotionTemplateRef | undefined) => { + if (!ref || ref.durationMs <= 0) return; + const index = motionInput(key, ref, W, H); + const v = `${key}_v`; + const a = `${key}_a`; + graph.push( + `[${index}:v]${[...fitFilters(W, H), "setsar=1", ...finishVideo(ref.durationMs)].join(",")}[${v}]`, + ); + silence(a, ref.durationMs); + program.push({ v, a, lengthMs: ref.durationMs }); + }; + bookend("intro", edl.intro); + program.push({ v: main.v, a: mainAudio, lengthMs: main.lengthMs }); + bookend("outro", edl.outro); + let videoLabel = main.v; + let programAudio = mainAudio; + if (program.length > 1) { + videoLabel = "prog_v"; + programAudio = "prog_a"; + graph.push( + `${program.map((p) => `[${p.v}][${p.a}]`).join("")}concat=n=${program.length}:v=1:a=1[prog_v][prog_a]`, + ); + } + + // Overlays on the output timeline. + const zones = profile.safeZones; + const margin = Math.round(W * 0.02); + const safeLeft = Math.round((zones.leftPct / 100) * W) + margin; + const safeRight = Math.round((zones.rightPct / 100) * W) + margin; + const safeTop = Math.round((zones.topPct / 100) * H) + margin; + const safeBottom = Math.round((zones.bottomPct / 100) * H) + margin; + const textColor = brandColor(kit?.colors.text, "white"); + const overlays = [...edl.overlays].sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)); + const imageOverlay = ( + path: string, + overlay: EdlOverlay, + widthPx: number, + opacity: number, + position: { x: string; y: string }, + ) => { + const index = inputs.add("-i", path); + const label = nextLabel("img"); + const out = nextLabel("ov"); + const alpha = opacity < 1 ? `,colorchannelmixer=aa=${num(opacity)}` : ""; + graph.push( + `[${index}:v]format=rgba,scale=${even(widthPx)}:-2${alpha}[${label}]`, + `[${videoLabel}][${label}]overlay=x=${escapeFilterValue(position.x)}:y=${escapeFilterValue( + position.y, + )}:enable=${enableBetween(overlay.atMs, overlay.atMs + overlay.durationMs)}[${out}]`, + ); + videoLabel = out; + }; + const boxPosition = (box: NormalizedBox) => ({ + x: String(Math.round(box.x * W)), + y: String(Math.round(box.y * H)), + }); + const cornerPosition = (corner: string) => ({ + x: corner.endsWith("left") + ? String(safeLeft) + : corner === "center" + ? "(main_w-overlay_w)/2" + : `main_w-overlay_w-${safeRight}`, + y: corner.startsWith("top") + ? String(safeTop) + : corner === "center" + ? "(main_h-overlay_h)/2" + : `main_h-overlay_h-${safeBottom}`, + }); + + for (const overlay of overlays) { + const endMs = overlay.atMs + overlay.durationMs; + const motionPath = input.motionClips.get(overlay.id); + if (overlay.motion || motionPath) { + const box = overlay.box; + const width = box ? even(box.w * W) : W; + const height = box ? even(box.h * H) : H; + const template = overlay.motion ?? { + templateId: "overlay", + params: {}, + durationMs: overlay.durationMs, + }; + const index = motionInput(overlay.id, template, width, height); + const label = nextLabel("mo"); + const out = nextLabel("ov"); + const position = box ? boxPosition(box) : { x: "0", y: "0" }; + graph.push( + `[${index}:v]format=yuva420p,scale=${width}:${height},setpts=PTS-STARTPTS+${sec(overlay.atMs)}/TB[${label}]`, + `[${videoLabel}][${label}]overlay=x=${position.x}:y=${position.y}:eof_action=pass:enable=${enableBetween( + overlay.atMs, + endMs, + )}[${out}]`, + ); + videoLabel = out; + continue; + } + if (overlay.kind === "watermark") { + const watermark = kit?.watermark; + if (!watermark || !input.files.watermarkPath) { + throw new Error(`overlay ${overlay.id} needs the brand watermark file`); + } + const width = overlay.box ? overlay.box.w * W : watermark.scale * W; + imageOverlay( + input.files.watermarkPath, + overlay, + width, + watermark.opacity, + overlay.box ? boxPosition(overlay.box) : cornerPosition(watermark.position), + ); + continue; + } + if (overlay.kind === "logo") { + if (!input.files.logoPath) throw new Error(`overlay ${overlay.id} needs a logo file`); + const width = overlay.box ? overlay.box.w * W : 0.15 * W; + imageOverlay( + input.files.logoPath, + overlay, + width, + 1, + overlay.box ? boxPosition(overlay.box) : cornerPosition("top_right"), + ); + continue; + } + if (overlay.kind === "progress_bar") { + const barHeight = Math.max(4, even(H * 0.006)); + const label = nextLabel("pb"); + const out = nextLabel("ov"); + const y = overlay.box ? Math.round(overlay.box.y * H) : H - safeBottom - barHeight; + const x = `-overlay_w+main_w*min(1,(t-${sec(overlay.atMs)})/${sec(Math.max(1, overlay.durationMs))})`; + graph.push( + `color=c=${brandColor(kit?.colors.accent, "white")}:s=${W}x${barHeight}:r=${num(F, 3)}:d=${sec(totalMs)}[${label}]`, + `[${videoLabel}][${label}]overlay=x=${escapeFilterValue(x)}:y=${y}:enable=${enableBetween( + overlay.atMs, + endMs, + )}[${out}]`, + ); + videoLabel = out; + continue; + } + // text, lower_third, cta_end_card + const text = overlay.text?.trim(); + if (!text) throw new Error(`overlay ${overlay.id} has no text and no motion clip`); + const lines = text.split(/\r?\n/).length; + const fontSize = overlay.box + ? Math.max(8, Math.round((overlay.box.h * H * 0.8) / lines)) + : Math.round(H * (overlay.kind === "cta_end_card" ? 0.05 : 0.04)); + const position = overlay.box + ? boxPosition(overlay.box) + : { + x: "(w-text_w)/2", + y: + overlay.kind === "lower_third" + ? `h-text_h-${safeBottom + Math.round(H * 0.08)}` + : overlay.kind === "cta_end_card" + ? "(h-text_h)/2" + : String(safeTop + Math.round(H * 0.05)), + }; + const font = pickFont( + kit, + input.files.fonts, + overlay.kind === "cta_end_card" ? "heading" : "body", + ); + const options = [ + ...(font ? [`fontfile=${escapeFilterValue(font)}`] : []), + `text=${escapeFilterText(text)}`, + `fontsize=${fontSize}`, + `fontcolor=${textColor}`, + `x=${escapeFilterValue(position.x)}`, + `y=${escapeFilterValue(position.y)}`, + ]; + if (overlay.kind !== "text") { + options.push( + "box=1", + `boxcolor=${brandColor(kit?.colors.primary, "black")}@0.85`, + `boxborderw=${Math.round(fontSize * 0.4)}`, + ); + } + options.push(`enable=${enableBetween(overlay.atMs, endMs)}`); + const out = nextLabel("ov"); + graph.push(`[${videoLabel}]drawtext=${options.join(":")}[${out}]`); + videoLabel = out; + } + + // Captions. + const captions = edl.captions; + if (captions && captions.mode !== "none") { + const cues = buildCaptionCues(outputCaptionWords(edl, input.transcripts), { + maxCharsPerLine: captions.maxCharsPerLine, + maxLines: profile.captions.maxLines, + }); + if (cues.length > 0) { + if (captions.mode === "burned" || captions.mode === "both") { + const preset = captionPreset(edl, kit); + const assPath = joinPath(workDir, "captions.ass"); + steps.push({ + kind: "write_file", + id: "captions-ass", + path: assPath, + content: toAss(cues, { + width: W, + height: H, + profile, + preset, + fallbackTextColor: kit?.colors.text ?? "#FFFFFF", + position: captions.position, + maxCharsPerLine: captions.maxCharsPerLine, + wordHighlight: captions.wordHighlight, + language: captions.language, + }), + }); + const font = pickFont(kit, input.files.fonts, "caption"); + const fontsDir = font ? `:fontsdir=${escapeFilterValue(dirName(font))}` : ""; + const out = nextLabel("cap"); + graph.push(`[${videoLabel}]ass=filename=${escapeFilterValue(assPath)}${fontsDir}[${out}]`); + videoLabel = out; + } + if (captions.mode === "sidecar" || captions.mode === "both") { + const srt = joinPath(workDir, "captions.srt"); + const vtt = joinPath(workDir, "captions.vtt"); + steps.push( + { kind: "write_file", id: "captions-srt", path: srt, content: toSrt(cues) }, + { kind: "write_file", id: "captions-vtt", path: vtt, content: toVtt(cues) }, + ); + outputs.push({ kind: "subtitles_srt", path: srt }, { kind: "subtitles_vtt", path: vtt }); + } + } + } + const finalVideo = input.videoEncoder.endsWith("_vaapi") + ? "format=nv12,hwupload" + : "format=yuv420p"; + graph.push(`[${videoLabel}]${finalVideo}[vout]`); + + // Audio: program (+ voice-over) is speech; clips marked duckUnderSpeech are compressed under it. + const speech: string[] = [programAudio]; + const ducked: string[] = []; + const beds: string[] = []; + const audioClips = [...edl.tracks.audio].sort( + (a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id), + ); + for (const clip of audioClips) { + const path = input.files.audio.get(clip.artifactId); + if (!path) throw new Error(`no file for audio clip ${clip.id}`); + const length = clip.outMs - clip.inMs; + const index = inputs.add("-ss", sec(clip.inMs), "-t", sec(length), "-i", path); + graph.push(`[${index}:a]${audioClipFilters(clip, length).join(",")}[ac_${clip.id}]`); + const label = `ac_${clip.id}`; + if (clip.kind === "voiceover") speech.push(label); + else if (clip.duckUnderSpeech) ducked.push(label); + else beds.push(label); + } + const mix = (labels: string[], out: string) => { + if (labels.length === 1) { + graph.push(`[${labels[0]}]anull[${out}]`); + } else { + graph.push( + `${labels.map((l) => `[${l}]`).join("")}amix=inputs=${labels.length}:duration=first:normalize=0[${out}]`, + ); + } + }; + const finalInputs: string[] = []; + if (ducked.length > 0) { + mix(speech, "speech"); + graph.push("[speech]asplit=2[speech_mix][speech_key]"); + mix(ducked, "duck_bus"); + graph.push( + "[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked]", + ); + finalInputs.push("speech_mix", "ducked"); + } else { + finalInputs.push(...speech); + } + finalInputs.push(...beds); + const targetLufs = edl.targetLufs ?? profile.audio.targetLufs; + const loudnorm = `loudnorm=I=${num(targetLufs, 1)}:TP=${num(profile.audio.truePeakDb, 1)}:LRA=11:linear=true`; + const mixed = finalInputs.length > 1 ? "mixed" : finalInputs[0]!; + if (finalInputs.length > 1) mix(finalInputs, "mixed"); + graph.push(`[${mixed}]${loudnorm},aresample=48000,${AUDIO_FORMAT}[aout]`); + + // Render step. + const graphText = graph.join(";"); + const graphArgs = + graphText.length > MAX_INLINE_GRAPH + ? ["-filter_complex_script", joinPath(workDir, "filtergraph.txt")] + : ["-filter_complex", graphText]; + if (graphText.length > MAX_INLINE_GRAPH) { + steps.push({ + kind: "write_file", + id: "filtergraph", + path: joinPath(workDir, "filtergraph.txt"), + content: graphText, + }); + } + const videoPath = joinPath(workDir, "output.mp4"); + steps.push({ + kind: "ffmpeg", + id: "render", + argv: [ + "-hide_banner", + "-nostdin", + "-y", + ...inputs.args, + ...graphArgs, + "-map", + "[vout]", + "-map", + "[aout]", + ...encodingArgs(input), + ...metadataArgs(edl.metadata.title), + "-t", + sec(totalMs), + "-movflags", + "+faststart", + videoPath, + ], + produces: [videoPath], + durationMs: totalMs, + }); + outputs.unshift({ kind: "video", path: videoPath }); + + if (profile.export.poster) { + const posterPath = joinPath(workDir, "poster.jpg"); + const at = clamp(input.posterAtMs ?? intro + mainMs / 3, 0, Math.max(0, totalMs - 1)); + steps.push({ + kind: "ffmpeg", + id: "poster", + argv: [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + sec(at), + "-i", + videoPath, + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + posterPath, + ], + produces: [posterPath], + durationMs: 0, + }); + outputs.push({ kind: "poster", path: posterPath }); + } + + return { quality: input.quality, workDir, steps, outputs }; +} + +function audioClipFilters(clip: EdlAudioClip, lengthMs: number): string[] { + const filters = ["asetpts=PTS-STARTPTS", "aresample=48000", AUDIO_FORMAT]; + if (clip.gainDb !== 0) filters.push(`volume=${num(clip.gainDb, 2)}dB`); + if (clip.fadeInMs > 0) filters.push(`afade=t=in:st=0:d=${sec(clip.fadeInMs)}`); + if (clip.fadeOutMs > 0) { + filters.push( + `afade=t=out:st=${sec(Math.max(0, lengthMs - clip.fadeOutMs))}:d=${sec(clip.fadeOutMs)}`, + ); + } + if (clip.atMs > 0) filters.push(`adelay=${clip.atMs}|${clip.atMs}`); + return filters; +} diff --git a/packages/media/src/edl.test.ts b/packages/media/src/edl.test.ts new file mode 100644 index 0000000..f72e9f9 --- /dev/null +++ b/packages/media/src/edl.test.ts @@ -0,0 +1,337 @@ +import type { EdlError } from "@rakazo/contracts"; +import { EdlErrorSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { EdlContext } from "./edl.js"; +import { edlDurationMs, locateOutputTime, locateOutputTimeAll, validateEdl } from "./edl.js"; +import { testContext, testEdl, testProfile } from "./test-fixtures.js"; + +/** Raw plan (pre-defaults) so cases can break it in any way. */ +function raw(patch: (plan: Record) => void): unknown { + const plan = JSON.parse(JSON.stringify(testEdl())) as Record; + patch(plan); + return plan; +} + +function codes(input: unknown, context: EdlContext = testContext()): EdlError["code"][] { + const result = validateEdl(input, context); + return result.ok ? [] : result.errors.map((e) => e.code); +} + +describe("validateEdl", () => { + it("accepts a valid plan and returns the parsed EDL", () => { + const result = validateEdl(testEdl(), testContext()); + expect(result.ok).toBe(true); + if (result.ok) expect(result.edl.tracks.video).toHaveLength(2); + }); + + const cases: Array<{ + code: EdlError["code"]; + input: unknown; + context?: Partial; + path?: string; + }> = [ + { code: "schema", input: raw((p) => (p.edlVersion = 2)), path: "edlVersion" }, + { + code: "schema", + input: raw((p) => (p.tracks.video[1].id = "c1")), + path: "tracks.video[1].id", + }, + { + code: "schema", + input: raw((p) => (p.tracks.video[0].id = "Bad Id;[x]")), + path: "tracks.video[0].id", + }, + { + code: "unknown_asset", + input: raw((p) => (p.tracks.video[0].assetId = "missing")), + path: "tracks.video[0].assetId", + }, + { + code: "unknown_clip", + input: raw((p) => { + p.transitions = [{ fromClipId: "c1", toClipId: "c9", kind: "cut", durationMs: 0 }]; + }), + path: "transitions[0].toClipId", + }, + { + code: "unknown_clip", + input: raw((p) => { + p.redactions.regions = [ + { + id: "r1", + clipId: "zz", + keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 0.1, h: 0.1 } }], + treatment: "blur", + }, + ]; + }), + }, + { + code: "unknown_artifact", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "sfx", artifactId: "nope", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "unknown_artifact", + input: raw((p) => (p.overlays = [{ id: "w", kind: "watermark", atMs: 0, durationMs: 1000 }])), + context: { brandKit: null }, + }, + { + code: "unknown_mark", + input: raw((p) => (p.redactions.marks = [{ markId: "mark-x", treatment: "blur" }])), + }, + { + code: "unknown_person", + input: raw((p) => { + p.redactions.faces = [{ id: "f", target: { personId: "person-x" }, mode: "blur" }]; + }), + }, + { + code: "unknown_track", + input: raw((p) => { + p.redactions.faces = [{ id: "f", target: { trackId: "track-x" }, mode: "pixelate" }]; + }), + }, + { + code: "unknown_template", + input: raw((p) => (p.intro = { templateId: "intro-none", durationMs: 1000 })), + path: "intro.templateId", + }, + { + code: "unknown_template", + input: raw((p) => (p.captions = { mode: "burned", presetId: "missing" })), + path: "captions.presetId", + }, + { + code: "range_out_of_asset", + input: raw((p) => (p.tracks.video[1].outMs = 12_000)), + path: "tracks.video[1].outMs", + }, + { + code: "range_out_of_asset", + input: raw((p) => (p.tracks.video[0].reframe = [{ tMs: 9000, cx: 0.5, cy: 0.5, scale: 1 }])), + }, + { + code: "range_empty", + input: raw((p) => (p.tracks.video[0].outMs = 0)), + }, + { + code: "range_empty", + input: raw( + (p) => (p.overlays = [{ id: "t", kind: "text", atMs: 0, durationMs: 0, text: "x" }]), + ), + }, + { + code: "overlap", + input: raw((p) => (p.tracks.video[1].atMs = 1500)), + path: "tracks.video[1].atMs", + }, + { + code: "overlap", + // A crossfade needs exactly its duration of overlap. + input: raw((p) => { + p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }]; + }), + }, + { + code: "overlap", + input: raw((p) => { + p.tracks.audio = [ + { id: "m1", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 0 }, + { id: "m2", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 2000 }, + ]; + }), + }, + { + code: "duration_below_min", + input: raw((p) => (p.output.durationHardMinMs = 5000)), + }, + { + code: "duration_below_min", + input: testEdl(), + context: { profile: testProfile({ duration: { hardMinMs: 6000, hardMaxMs: 60_000 } }) }, + }, + { + code: "duration_above_max", + input: raw((p) => (p.output.durationHardMaxMs = 3000)), + }, + { + code: "duration_above_max", + input: testEdl(), + context: { profile: testProfile({ duration: { hardMinMs: 0, hardMaxMs: 3000 } }) }, + }, + { + code: "overlay_outside_safe_zone", + input: raw((p) => { + p.overlays = [ + { + id: "t", + kind: "text", + atMs: 0, + durationMs: 1000, + text: "Hi", + box: { x: 0.1, y: 0.85, w: 0.5, h: 0.1 }, + }, + ]; + }), + path: "overlays[0].box", + }, + { + code: "paid_only_segment", + input: testEdl(), + context: { paidOnly: [{ assetId: "asset-a", startMs: 6500, endMs: 9000 }] }, + path: "tracks.video[1]", + }, + { + code: "paid_only_segment", + input: testEdl({ output: { ...testEdl().output, profileId: "example-paid-video" } }), + context: { + profile: testProfile({ id: "example-paid-video", kind: "paid_video" }), + paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 100 }], + goal: "teaser", + }, + }, + { + code: "profile_mismatch", + input: raw((p) => (p.output.profileId = "other-profile")), + }, + { + code: "profile_mismatch", + input: raw((p) => { + p.output.width = 1920; + p.output.height = 1080; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "vo-1", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-1", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + context: { profile: testProfile({ audio: { music: "platform_library_only" } }) }, + }, + ]; + + it.each(cases)("rejects $code ($path)", ({ code, input, context, path }) => { + const result = validateEdl(input, testContext(context)); + expect(result.ok).toBe(false); + if (result.ok) return; + for (const error of result.errors) expect(EdlErrorSchema.safeParse(error).success).toBe(true); + const match = result.errors.find((e) => e.code === code); + expect(match, JSON.stringify(result.errors)).toBeDefined(); + if (path) expect(match!.path).toBe(path); + }); + + it("covers every error code", () => { + const covered = new Set(cases.map((c) => c.code)); + expect([...covered].sort()).toEqual([...EdlErrorSchema.shape.code.options].sort()); + }); + + it("allows paid-only material in a full release for a paid profile and any music where allowed", () => { + const paid = testProfile({ + id: "example-paid-video", + kind: "paid_video", + audio: { music: "any" }, + }); + const plan = raw((p) => { + p.output.profileId = "example-paid-video"; + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }); + expect( + codes( + plan, + testContext({ profile: paid, paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 9000 }] }), + ), + ).toEqual([]); + }); + + it("accepts photos without a duration and a crossfade with the right overlap", () => { + const plan = raw((p) => { + p.tracks.video[0].assetId = "photo-1"; + p.tracks.video[1].atMs = 1500; + p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }]; + }); + expect(codes(plan)).toEqual([]); + }); + + it("reports every failure at once", () => { + const plan = raw((p) => { + p.tracks.video[0].assetId = "missing"; + p.tracks.video[1].outMs = 4000; + p.redactions.marks = [{ markId: "mark-x", treatment: "cover" }]; + }); + expect(codes(plan)).toEqual(["unknown_asset", "range_empty", "unknown_mark"]); + }); +}); + +describe("timeline", () => { + const edl = testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1000 }, + outro: { templateId: "outro-simple", durationMs: 500 }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" }, + { + id: "c2", + assetId: "asset-b", + inMs: 1000, + outMs: 5000, + atMs: 1500, + speed: 2, + reframe: "none", + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "dip", durationMs: 500 }], + }); + + it("measures intro, speed, transition overlap and outro", () => { + expect(edlDurationMs(edl)).toBe(1000 + 1500 + 2000 + 500); + expect(edlDurationMs(testEdl())).toBe(4000); + }); + + it("maps output time back to clip and source time", () => { + expect(locateOutputTime(edl, 500)).toBeNull(); + expect(locateOutputTime(edl, 1000)).toEqual({ clipId: "c1", assetId: "asset-a", sourceMs: 0 }); + expect(locateOutputTime(edl, 2200)).toEqual({ + clipId: "c1", + assetId: "asset-a", + sourceMs: 1200, + }); + // Inside the transition both clips are visible; the incoming one wins. + expect(locateOutputTimeAll(edl, 2700).map((h) => h.clipId)).toEqual(["c1", "c2"]); + expect(locateOutputTime(edl, 2700)).toEqual({ + clipId: "c2", + assetId: "asset-b", + sourceMs: 1400, + }); + expect(locateOutputTime(edl, 4400)).toEqual({ + clipId: "c2", + assetId: "asset-b", + sourceMs: 4800, + }); + expect(locateOutputTime(edl, 4600)).toBeNull(); + }); +}); diff --git a/packages/media/src/edl.ts b/packages/media/src/edl.ts new file mode 100644 index 0000000..384f3d3 --- /dev/null +++ b/packages/media/src/edl.ts @@ -0,0 +1,426 @@ +import type { + AspectRatio, + BrandKit, + Edl, + EdlError, + EdlTransition, + EdlVideoClip, + MediaAssetKind, + MediaProjectGoal, + NormalizedBox, + PlatformProfile, +} from "@rakazo/contracts"; +import { EdlSchema } from "@rakazo/contracts"; + +/* + * Timeline model shared by validation, compilation, gate and auto-fix: + * + * - The output is intro + main program + outro, played back to back. + * - `tracks.video[].atMs` is a position on the main program, which starts + * after the intro. A clip lasts (outMs - inMs) / speed. + * - Clips may only overlap where a crossfade, dip or whip transition joins + * them, and then by exactly the transition duration. Cuts and match cuts + * never overlap. + * - Overlays and audio clips are positioned on the output timeline (intro + * included), so music can run under the intro. + * - Reframe keyframes are in source time of the clip's asset. + */ + +export interface EdlContext { + profile: PlatformProfile; + brandKit: BrandKit | null; + assets: Map< + string, + { kind: MediaAssetKind; durationMs: number | null; width: number | null; height: number | null } + >; + /** Artifact ids the plan may reference (music, voice-over, fonts, logos). */ + artifactIds: Set; + /** Music artifacts and whether their license allows commercial use. */ + music: Map; + markIds: Set; + personIds: Set; + faceTrackIds: Set; + templateIds: Set; + /** Segments the user reserved for the paid release. */ + paidOnly: Array<{ assetId: string; startMs: number; endMs: number }>; + /** Project goal; a teaser never uses paid-only segments, whatever the profile. */ + goal?: MediaProjectGoal; +} + +export type EdlValidation = { ok: true; edl: Edl } | { ok: false; errors: EdlError[] }; + +/** Transitions rendered as a timed blend (xfade). Cuts and match cuts are plain concatenation. */ +export const BLENDED_TRANSITIONS: ReadonlySet = new Set([ + "crossfade", + "dip", + "whip", +]); + +export const ASPECT_RATIO_VALUES: Record = { + "9:16": 9 / 16, + "4:5": 4 / 5, + "1:1": 1, + "16:9": 16 / 9, + "4:3": 4 / 3, + "21:9": 21 / 9, +}; + +const ASPECT_TOLERANCE = 0.01; + +/** Whether width × height matches one of the allowed aspect ratios within 1 %. */ +export function matchesAspect(width: number, height: number, allowed: AspectRatio[]): boolean { + const actual = width / height; + return allowed.some( + (ratio) => + Math.abs(actual - ASPECT_RATIO_VALUES[ratio]) / ASPECT_RATIO_VALUES[ratio] <= + ASPECT_TOLERANCE, + ); +} + +/** Output length of one video clip after speed change. */ +export function clipLengthMs(clip: Pick): number { + return Math.max(0, Math.round((clip.outMs - clip.inMs) / clip.speed)); +} + +/** End of the main program (excluding intro and outro). */ +export function mainProgramMs(edl: Edl): number { + let end = 0; + for (const clip of edl.tracks.video) end = Math.max(end, clip.atMs + clipLengthMs(clip)); + return end; +} + +export function introMs(edl: Edl): number { + return edl.intro?.durationMs ?? 0; +} + +/** Output length in ms including intro, outro, speed, and transition overlap. */ +export function edlDurationMs(edl: Edl): number { + return introMs(edl) + mainProgramMs(edl) + (edl.outro?.durationMs ?? 0); +} + +/** Every clip visible at an output timestamp (two during a blended transition), earliest first. */ +export function locateOutputTimeAll( + edl: Edl, + outputMs: number, +): Array<{ clipId: string; assetId: string; sourceMs: number }> { + const t = outputMs - introMs(edl); + if (t < 0) return []; + return [...edl.tracks.video] + .filter((clip) => t >= clip.atMs && t < clip.atMs + clipLengthMs(clip)) + .sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)) + .map((clip) => ({ + clipId: clip.id, + assetId: clip.assetId, + sourceMs: Math.min(clip.outMs, Math.round(clip.inMs + (t - clip.atMs) * clip.speed)), + })); +} + +/** + * Map an output timestamp back to the clip and source time it shows. During a + * blended transition the incoming clip wins; intro and outro map to null. + */ +export function locateOutputTime( + edl: Edl, + outputMs: number, +): { clipId: string; assetId: string; sourceMs: number } | null { + return locateOutputTimeAll(edl, outputMs).at(-1) ?? null; +} + +/** Whether a normalized box lies inside the profile's safe zones. */ +export function boxInsideSafeZone(box: NormalizedBox, profile: PlatformProfile): boolean { + const epsilon = 1e-6; + const zones = profile.safeZones; + return ( + box.x >= zones.leftPct / 100 - epsilon && + box.y >= zones.topPct / 100 - epsilon && + box.x + box.w <= 1 - zones.rightPct / 100 + epsilon && + box.y + box.h <= 1 - zones.bottomPct / 100 + epsilon + ); +} + +function formatPath(path: ReadonlyArray): string { + let out = ""; + for (const part of path) { + if (typeof part === "number") out += `[${part}]`; + else out += out ? `.${String(part)}` : String(part); + } + return out || "(root)"; +} + +/** Video clips in timeline order, ties broken by id for determinism. */ +export function sortedVideoClips(edl: Edl): EdlVideoClip[] { + return [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)); +} + +/** + * Parse with EdlSchema, then check references, ranges, overlaps, durations, + * safe zones, paid-only use (teaser-style profiles), and music licensing. + * Every failure is reported; nothing is silently corrected. + */ +export function validateEdl(input: unknown, context: EdlContext): EdlValidation { + const parsed = EdlSchema.safeParse(input); + if (!parsed.success) { + return { + ok: false, + errors: parsed.error.issues.map((issue) => ({ + code: "schema", + path: formatPath(issue.path), + message: issue.message, + })), + }; + } + const edl = parsed.data; + const errors: EdlError[] = []; + const fail = (code: EdlError["code"], path: string, message: string) => + errors.push({ code, path, message }); + const { profile } = context; + + if (edl.output.profileId !== profile.id) { + fail("profile_mismatch", "output.profileId", `plan targets ${edl.output.profileId}`); + } + if (!matchesAspect(edl.output.width, edl.output.height, profile.video.aspectRatios)) { + fail( + "profile_mismatch", + "output", + `${edl.output.width}x${edl.output.height} is not one of ${profile.video.aspectRatios.join(", ")}`, + ); + } + if (edl.output.fps > profile.video.fps * 1.001) { + fail("profile_mismatch", "output.fps", `fps above profile maximum ${profile.video.fps}`); + } + + const seenIds = new Map(); + const claimId = (id: string, path: string) => { + const previous = seenIds.get(id); + if (previous) fail("schema", path, `id ${id} already used at ${previous}`); + else seenIds.set(id, path); + }; + + const clipsById = new Map(); + edl.tracks.video.forEach((clip, index) => { + const path = `tracks.video[${index}]`; + claimId(clip.id, `${path}.id`); + clipsById.set(clip.id, clip); + if (clip.outMs <= clip.inMs) fail("range_empty", path, "outMs must be after inMs"); + const asset = context.assets.get(clip.assetId); + if (!asset) { + fail("unknown_asset", `${path}.assetId`, `asset ${clip.assetId} is not available`); + } else if (asset.durationMs !== null && asset.kind !== "photo") { + if (clip.outMs > asset.durationMs) { + fail( + "range_out_of_asset", + `${path}.outMs`, + `outMs ${clip.outMs} exceeds asset duration ${asset.durationMs}`, + ); + } + if (clip.inMs >= asset.durationMs) { + fail("range_out_of_asset", `${path}.inMs`, `inMs ${clip.inMs} is past the asset end`); + } + } + if (Array.isArray(clip.reframe)) { + clip.reframe.forEach((keyframe, k) => { + if (keyframe.tMs < clip.inMs || keyframe.tMs > clip.outMs) { + fail( + "range_out_of_asset", + `${path}.reframe[${k}].tMs`, + "reframe keyframe lies outside the clip's source range", + ); + } + }); + } + if ( + (profile.kind === "free_social" || context.goal === "teaser") && + context.paidOnly.some( + (segment) => + segment.assetId === clip.assetId && + segment.startMs < clip.outMs && + clip.inMs < segment.endMs, + ) + ) { + fail("paid_only_segment", path, "clip uses material reserved for the paid release"); + } + }); + + // Transitions: known clips, adjacent in time, at most one in and one out per clip. + const blendBetween = new Map(); + const outgoing = new Set(); + const incoming = new Set(); + const order = sortedVideoClips(edl); + const nextOf = new Map(); + for (let i = 0; i + 1 < order.length; i++) nextOf.set(order[i]!.id, order[i + 1]!.id); + edl.transitions.forEach((transition, index) => { + const path = `transitions[${index}]`; + const from = clipsById.get(transition.fromClipId); + const to = clipsById.get(transition.toClipId); + if (!from) fail("unknown_clip", `${path}.fromClipId`, `no clip ${transition.fromClipId}`); + if (!to) fail("unknown_clip", `${path}.toClipId`, `no clip ${transition.toClipId}`); + if (!from || !to) return; + if (nextOf.get(from.id) !== to.id) { + fail("schema", path, "a transition must join a clip to the clip that follows it"); + return; + } + if (outgoing.has(from.id) || incoming.has(to.id)) { + fail("schema", path, "a clip can have only one transition on each side"); + return; + } + outgoing.add(from.id); + incoming.add(to.id); + if (BLENDED_TRANSITIONS.has(transition.kind) && transition.durationMs > 0) { + const shortest = Math.min(clipLengthMs(from), clipLengthMs(to)); + if (transition.durationMs >= shortest) { + fail("overlap", `${path}.durationMs`, "transition is longer than one of its clips"); + return; + } + blendBetween.set(`${from.id}>${to.id}`, transition.durationMs); + } + }); + + // Overlaps on the video track: only blended transitions may overlap, by exactly their length. + for (let i = 0; i + 1 < order.length; i++) { + const a = order[i]!; + const b = order[i + 1]!; + const overlap = a.atMs + clipLengthMs(a) - b.atMs; + const blend = blendBetween.get(`${a.id}>${b.id}`); + const path = `tracks.video[${edl.tracks.video.indexOf(b)}].atMs`; + if (blend !== undefined) { + if (Math.abs(overlap - blend) > 1) { + fail( + "overlap", + path, + `clips ${a.id} and ${b.id} must overlap by the ${blend} ms transition`, + ); + } + } else if (overlap > 1) { + fail("overlap", path, `clip ${b.id} starts ${overlap} ms before ${a.id} ends`); + } + } + for (let i = 0; i + 2 < order.length; i++) { + const a = order[i]!; + const c = order[i + 2]!; + if (a.atMs + clipLengthMs(a) > c.atMs + 1) { + fail("overlap", `tracks.video[${edl.tracks.video.indexOf(c)}].atMs`, "three clips overlap"); + } + } + + // Audio clips. + const audioByKind = new Map>(); + edl.tracks.audio.forEach((clip, index) => { + const path = `tracks.audio[${index}]`; + claimId(clip.id, `${path}.id`); + if (clip.outMs <= clip.inMs) fail("range_empty", path, "outMs must be after inMs"); + if (!context.artifactIds.has(clip.artifactId)) { + fail( + "unknown_artifact", + `${path}.artifactId`, + `artifact ${clip.artifactId} is not available`, + ); + } + if (clip.fadeInMs + clip.fadeOutMs > clip.outMs - clip.inMs && clip.outMs > clip.inMs) { + fail("range_empty", path, "fades are longer than the clip"); + } + if (clip.kind === "music") { + if (profile.audio.music === "platform_library_only") { + fail( + "music_license", + `${path}.artifactId`, + "this platform only allows music added from its own library", + ); + } else if (profile.audio.music === "licensed_only") { + const license = context.music.get(clip.artifactId); + if (!license?.commercialUse) { + fail( + "music_license", + `${path}.artifactId`, + license ? "music is not licensed for commercial use" : "music license is unknown", + ); + } + } + } + if (clip.kind !== "sfx") { + const list = audioByKind.get(clip.kind) ?? []; + list.push({ start: clip.atMs, end: clip.atMs + (clip.outMs - clip.inMs), index }); + audioByKind.set(clip.kind, list); + } + }); + for (const list of audioByKind.values()) { + list.sort((a, b) => a.start - b.start || a.index - b.index); + for (let i = 0; i + 1 < list.length; i++) { + if (list[i]!.end > list[i + 1]!.start) { + fail("overlap", `tracks.audio[${list[i + 1]!.index}].atMs`, "audio clips overlap"); + } + } + } + + // Templates. + const checkTemplate = (templateId: string, path: string) => { + if (!context.templateIds.has(templateId)) { + fail("unknown_template", path, `template ${templateId} is not available`); + } + }; + if (edl.intro) checkTemplate(edl.intro.templateId, "intro.templateId"); + if (edl.outro) checkTemplate(edl.outro.templateId, "outro.templateId"); + + // Overlays. + edl.overlays.forEach((overlay, index) => { + const path = `overlays[${index}]`; + claimId(overlay.id, `${path}.id`); + if (overlay.durationMs <= 0) + fail("range_empty", `${path}.durationMs`, "overlay has no duration"); + if (overlay.motion) checkTemplate(overlay.motion.templateId, `${path}.motion.templateId`); + if (overlay.box && !boxInsideSafeZone(overlay.box, profile)) { + fail("overlay_outside_safe_zone", `${path}.box`, "overlay leaves the profile safe zone"); + } + if (overlay.kind === "watermark" && !context.brandKit?.watermark) { + fail("unknown_artifact", path, "the brand kit has no watermark"); + } + if ((overlay.kind === "text" || overlay.kind === "lower_third") && !overlay.text?.trim()) { + if (!overlay.motion) fail("range_empty", `${path}.text`, "text overlay has no text"); + } + }); + + // Captions preset. + const presetId = edl.captions?.presetId; + if (presetId && context.brandKit) { + if (!context.brandKit.captionPresets.some((preset) => preset.id === presetId)) { + fail("unknown_template", "captions.presetId", `caption preset ${presetId} is not in the kit`); + } + } + + // Redactions. + edl.redactions.faces.forEach((redaction, index) => { + const path = `redactions.faces[${index}]`; + claimId(redaction.id, `${path}.id`); + const target = redaction.target; + if ("trackId" in target && !context.faceTrackIds.has(target.trackId)) { + fail("unknown_track", `${path}.target.trackId`, `face track ${target.trackId} is unknown`); + } + if ("personId" in target && !context.personIds.has(target.personId)) { + fail("unknown_person", `${path}.target.personId`, `person ${target.personId} is unknown`); + } + }); + edl.redactions.marks.forEach((mark, index) => { + if (!context.markIds.has(mark.markId)) { + fail("unknown_mark", `redactions.marks[${index}].markId`, `mark ${mark.markId} is unknown`); + } + }); + edl.redactions.regions.forEach((region, index) => { + const path = `redactions.regions[${index}]`; + claimId(region.id, `${path}.id`); + if (!clipsById.has(region.clipId)) { + fail("unknown_clip", `${path}.clipId`, `no clip ${region.clipId}`); + } + }); + + // Duration against both the plan's and the profile's hard bounds. + const duration = edlDurationMs(edl); + const minMs = Math.max(edl.output.durationHardMinMs, profile.duration.hardMinMs); + const maxMs = Math.min(edl.output.durationHardMaxMs, profile.duration.hardMaxMs); + if (duration < minMs) { + fail("duration_below_min", "output", `duration ${duration} ms is below ${minMs} ms`); + } + if (duration > maxMs) { + fail("duration_above_max", "output", `duration ${duration} ms is above ${maxMs} ms`); + } + + return errors.length > 0 ? { ok: false, errors } : { ok: true, edl }; +} diff --git a/packages/media/src/gate.test.ts b/packages/media/src/gate.test.ts new file mode 100644 index 0000000..acb75e3 --- /dev/null +++ b/packages/media/src/gate.test.ts @@ -0,0 +1,531 @@ +import type { FfprobeReport, GateReport } from "@rakazo/contracts"; +import { GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { edlDurationMs } from "./edl.js"; +import type { GateInput } from "./gate.js"; +import { + checkCopy, + checkTechnical, + evaluateGate, + findMetadataLeaks, + METADATA_ALLOWED_TAGS, + planAutoFix, +} from "./gate.js"; +import { testEdl, testPaidProfile, testProfile } from "./test-fixtures.js"; + +const cleanTags = { + "format.major_brand": "isom", + "format.minor_version": "512", + "format.compatible_brands": "isomiso2avc1mp41", + "format.title": "Example", + "stream.0.language": "und", + "stream.0.handler_name": "VideoHandler", + "stream.0.vendor_id": "[0][0][0][0]", + "stream.0.encoder": "Lavc libx264", + "stream.1.handler_name": "SoundHandler", +}; + +function probe(overrides: Partial = {}): FfprobeReport { + return { + durationMs: 4000, + width: 1080, + height: 1920, + fps: 30, + videoCodec: "h264", + audioCodec: "aac", + sizeBytes: 2_000_000, + integratedLufs: -14.3, + truePeakDb: -1.2, + tags: cleanTags, + ...overrides, + }; +} + +function gateInput(overrides: Partial = {}): GateInput { + return { + profile: testProfile(), + edl: testEdl(), + ffprobe: probe(), + content: [ + { tMs: 0, scores: { explicit_nudity: 0.01 } }, + { tMs: 500, scores: { explicit_nudity: 0.02, violence: 0.1 } }, + ], + faces: [], + trackingGaps: [], + marks: [], + people: [], + copy: { title: "A day out", caption: "Short caption #one", tags: ["two"] }, + aiAlterations: [], + bundleHasDisclosure: false, + watermarkBox: { x: 0.7, y: 0.12, w: 0.15, h: 0.05 }, + sampledFrames: 8, + autoFixRound: 0, + now: new Date("2026-10-01T12:00:00Z"), + ...overrides, + }; +} + +const blockCodes = (report: GateReport) => + report.issues.filter((i) => i.severity === "block").map((i) => i.code); + +describe("metadata", () => { + it("allows only harmless container tags", () => { + expect(METADATA_ALLOWED_TAGS).toContain("title"); + expect(findMetadataLeaks(cleanTags)).toEqual([]); + expect( + findMetadataLeaks({ + ...cleanTags, + "format.creation_time": "2026-01-01T00:00:00Z", + "format.location": "+00.0000+000.0000/", + "format.com.apple.quicktime.make": "Example", + "format.com.apple.quicktime.location.ISO6709": "+00+000/", + "format.make": "Example", + "format.model": "Example", + "format.software": "Example 1.0", + "stream.0.creation_time": "2026-01-01T00:00:00Z", + GPSLatitude: "0", + "format.encoder": "Lavf60.16.100", + "stream:0:encoder": "x264 core 164", + }), + ).toEqual([ + "GPSLatitude", + "format.com.apple.quicktime.location.ISO6709", + "format.com.apple.quicktime.make", + "format.creation_time", + "format.encoder", + "format.location", + "format.make", + "format.model", + "format.software", + "stream.0.creation_time", + "stream:0:encoder", + ]); + expect(findMetadataLeaks({ title: "x", major_brand: "isom" })).toEqual([]); + }); +}); + +describe("evaluateGate", () => { + it("passes a clean render and produces a schema-valid report", () => { + const report = evaluateGate(gateInput()); + expect(report.issues).toEqual([]); + expect(report.passed).toBe(true); + expect(GateReportSchema.parse(report)).toEqual(report); + expect(report.checkedAt).toBe("2026-10-01T12:00:00.000Z"); + }); + + it("blocks technical problems", () => { + const edl = testEdl(); + expect(checkTechnical(testProfile(), edl, probe())).toEqual([]); + const cases: Array<[Partial, string]> = [ + [{ width: 1920, height: 1080 }, "resolution_mismatch"], + [{ width: 1920, height: 1080 }, "aspect_not_allowed"], + [{ fps: 25 }, "fps_mismatch"], + [{ durationMs: 70_000 }, "duration_too_long"], + [{ durationMs: 500 }, "duration_too_short"], + [{ sizeBytes: 600_000_000 }, "file_too_large"], + [{ videoCodec: "vp9" }, "codec_mismatch"], + [{ integratedLufs: -20 }, "loudness_off_target"], + [{ integratedLufs: null }, "loudness_unmeasured"], + [{ truePeakDb: 0 }, "true_peak_exceeded"], + [{ tags: { ...cleanTags, "format.creation_time": "2020" } }, "metadata_leak"], + ]; + for (const [patch, code] of cases) { + const report = evaluateGate(gateInput({ ffprobe: probe(patch) })); + expect(report.passed, code).toBe(false); + expect(blockCodes(report), code).toContain(code); + } + const drift = checkTechnical(testProfile(), edl, probe({ durationMs: 4500 })); + expect(drift).toEqual([expect.objectContaining({ code: "duration_drift", severity: "warn" })]); + expect( + checkTechnical(testProfile(), edl, probe({ audioCodec: null, integratedLufs: null })), + ).toEqual([]); + }); + + it("blocks classifier hits with timestamps, category and score", () => { + const report = evaluateGate( + gateInput({ + content: [ + { tMs: 0, scores: { explicit_nudity: 0.05 } }, + { tMs: 1500, scores: { explicit_nudity: 0.4 } }, + { tMs: 2000, scores: { explicit_nudity: 0.6 } }, + { tMs: 3500, scores: { violence: 0.9, drugs: 1 } }, + ], + }), + ); + expect(report.passed).toBe(false); + expect(report.issues).toEqual([ + expect.objectContaining({ + check: "content", + code: "content_threshold", + category: "explicit_nudity", + atMs: 1500, + endMs: 2000, + score: 0.6, + }), + expect.objectContaining({ category: "violence", atMs: 3500, severity: "block" }), + ]); + }); + + it("only blocks above the ratio when the profile says so", () => { + const profile = testProfile({ + content: { thresholds: { violence: 0.5 }, blockOn: "ratio", blockRatio: 0.25 }, + }); + const samples = Array.from({ length: 10 }, (_, i) => ({ + tMs: i * 500, + scores: { violence: i < 2 ? 0.9 : 0 }, + })); + const below = evaluateGate(gateInput({ profile, content: samples })); + expect(below.passed).toBe(true); + expect(below.issues[0]).toMatchObject({ severity: "warn", category: "violence" }); + const above = samples.map((s, i) => ({ ...s, scores: { violence: i < 4 ? 0.9 : 0 } })); + expect(evaluateGate(gateInput({ profile, content: above })).passed).toBe(false); + }); + + it("blocks anonymization gaps, uncovered and unknown faces, and visible marks", () => { + const people = [ + { id: "person-1", anonymizeByDefault: "blur", consentRecordedAt: null, ageVerifiedAt: null }, + { id: "person-2", anonymizeByDefault: "none", consentRecordedAt: null, ageVerifiedAt: null }, + ]; + const box = { x: 0.4, y: 0.2, w: 0.1, h: 0.1 }; + const report = evaluateGate( + gateInput({ + people, + faces: [ + { tMs: 100, box, personId: "person-1", coverage: 1 }, + { tMs: 200, box, personId: "person-1", coverage: 0.9 }, + { tMs: 300, box, personId: "person-2", coverage: 0 }, + { tMs: 400, box, personId: null, coverage: 0 }, + ], + trackingGaps: [ + { startMs: 1000, endMs: 1400, personId: "person-1" }, + { startMs: 2000, endMs: 2600, personId: "person-1" }, + ], + marks: [ + { markId: "mark-1", tMs: 100, residualVisibility: 0.01 }, + { markId: "mark-1", tMs: 300, residualVisibility: 0.3 }, + ], + }), + ); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([ + expect.objectContaining({ code: "face_not_covered", personId: "person-1", atMs: 200 }), + expect.objectContaining({ code: "unknown_face", atMs: 400 }), + expect.objectContaining({ + code: "tracking_gap", + atMs: 2000, + endMs: 2600, + personId: "person-1", + }), + expect.objectContaining({ code: "mark_visible", markId: "mark-1", atMs: 300 }), + ]); + // Without protected people an unmatched face is not an anonymization issue. + const open = evaluateGate(gateInput({ faces: [{ tMs: 0, box, personId: null, coverage: 0 }] })); + expect(open.passed).toBe(true); + }); + + it("requires consent and age verification only where the profile does", () => { + const people = [ + { + id: "p-ok", + anonymizeByDefault: "none", + consentRecordedAt: "2026-01-01T00:00:00Z", + ageVerifiedAt: "2026-01-01T00:00:00Z", + }, + { + id: "p-no-consent", + anonymizeByDefault: "none", + consentRecordedAt: null, + ageVerifiedAt: "2026-01-01T00:00:00Z", + }, + { + id: "p-no-age", + anonymizeByDefault: "none", + consentRecordedAt: "2026-01-01T00:00:00Z", + ageVerifiedAt: null, + }, + ]; + const faces = [{ tMs: 1000, box: { x: 0, y: 0, w: 0.1, h: 0.1 }, personId: null, coverage: 0 }]; + expect(evaluateGate(gateInput({ people, faces })).passed).toBe(true); + const profile = testPaidProfile(); + const edl = testEdl({ + output: { ...testEdl().output, profileId: profile.id, width: 1920, height: 1080 }, + }); + const report = evaluateGate( + gateInput({ profile, edl, people, faces, ffprobe: probe({ width: 1920, height: 1080 }) }), + ); + expect(report.issues.filter((i) => i.check === "consent")).toEqual([ + expect.objectContaining({ code: "age_unverified", personId: "p-no-age" }), + expect.objectContaining({ code: "consent_missing", personId: "p-no-consent" }), + expect.objectContaining({ code: "unknown_performer", atMs: 1000 }), + ]); + expect(report.passed).toBe(false); + }); + + it("requires the AI label when the profile does and warns about the watermark", () => { + const report = evaluateGate( + gateInput({ + aiAlterations: ["synthetic_identity"], + watermarkBox: { x: 0.9, y: 0.01, w: 0.1, h: 0.05 }, + }), + ); + expect(report.issues).toEqual([ + expect.objectContaining({ check: "disclosure", code: "ai_label_missing", severity: "block" }), + expect.objectContaining({ + check: "brand", + code: "watermark_outside_safe_zone", + severity: "warn", + }), + ]); + expect( + evaluateGate(gateInput({ aiAlterations: ["inpaint"], bundleHasDisclosure: true })).passed, + ).toBe(true); + const edl = testEdl({ metadata: { aiAltered: true, aiAlterations: ["voice_disguise"] } }); + expect(blockCodes(evaluateGate(gateInput({ edl })))).toEqual(["ai_label_missing"]); + const relaxed = testProfile({ disclosure: { aiAlteredLabelRequired: false } }); + expect(evaluateGate(gateInput({ profile: relaxed, edl })).passed).toBe(true); + }); +}); + +describe("checkCopy", () => { + const profile = testProfile({ + text: { + titleMaxChars: 20, + captionMaxChars: 60, + hashtagLimit: 2, + linkPolicy: "bio_only", + bannedPatterns: [ + { pattern: "free\\s+money", reason: "misleading claim" }, + { pattern: "([a-z", reason: "broken pattern" }, + ], + }, + }); + + it("checks banned patterns case-insensitively and reports invalid ones without crashing", () => { + const issues = checkCopy(profile, { + title: "FREE Money", + caption: null, + tags: ["free money"], + }); + expect(issues.map((i) => [i.code, i.message])).toEqual([ + ["banned_pattern", "title: misleading claim"], + ["banned_pattern", "tags[0]: misleading claim"], + ["invalid_banned_pattern", "profile pattern cannot be checked: broken pattern"], + ]); + }); + + it("checks lengths, hashtags and links", () => { + const codes = (copy: { title: string | null; caption: string | null; tags: string[] }) => + checkCopy(profile, copy) + .map((i) => i.code) + .filter((c) => c !== "invalid_banned_pattern"); + expect(codes({ title: "x".repeat(21), caption: "y".repeat(61), tags: [] })).toEqual([ + "title_too_long", + "caption_too_long", + ]); + expect(codes({ title: null, caption: "#a #b", tags: ["#c"] })).toEqual(["too_many_hashtags"]); + expect(codes({ title: null, caption: "#a #A", tags: ["a"] })).toEqual([]); + expect(codes({ title: null, caption: "see https://example.com/x", tags: [] })).toEqual([ + "link_bio_only", + ]); + expect(codes({ title: null, caption: "visit example.shop today", tags: [] })).toEqual([ + "link_bio_only", + ]); + expect(codes({ title: "Link in bio", caption: "Ready? Go.", tags: [] })).toEqual([]); + const forbidden = testProfile({ + text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "forbidden" }, + }); + expect( + checkCopy(forbidden, { title: "www.example.com", caption: null, tags: [] })[0]!.code, + ).toBe("link_forbidden"); + const allowed = testProfile({ + text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "allowed" }, + }); + expect(checkCopy(allowed, { title: null, caption: "https://example.com", tags: [] })).toEqual( + [], + ); + }); +}); + +describe("planAutoFix", () => { + const edl = testEdl({ + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 0, + outMs: 2000, + atMs: 0, + reframe: "auto", + segmentId: "s1", + }, + { + id: "c2", + assetId: "asset-a", + inMs: 5000, + outMs: 7000, + atMs: 2000, + reframe: "auto", + segmentId: "s2", + }, + { + id: "c3", + assetId: "asset-a", + inMs: 8000, + outMs: 10_000, + atMs: 4000, + reframe: "auto", + segmentId: "s3", + }, + ], + }, + transitions: [{ fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }], + overlays: [ + { id: "wm", kind: "watermark", atMs: 0, durationMs: 6000 }, + { id: "cta", kind: "cta_end_card", atMs: 4500, durationMs: 1500, text: "Link in bio" }, + ], + redactions: { + faces: [{ id: "f1", target: { personId: "person-1" }, mode: "blur", paddingPct: 20 }], + marks: [{ markId: "mark-1", treatment: "inpaint" }], + }, + metadata: { aiAltered: true, aiAlterations: ["inpaint"] }, + }); + const report = (issues: GateReport["issues"], round = 0): GateReport => ({ + profileId: "example-short-vertical", + passed: !issues.some((i) => i.severity === "block"), + checkedAt: "2026-10-01T12:00:00.000Z", + sampledFrames: 12, + issues, + autoFixRound: round, + }); + const contentHit = (atMs: number, endMs = atMs) => + ({ + check: "content", + severity: "block", + code: "content_threshold", + message: "violence", + atMs, + endMs, + category: "violence", + score: 0.9, + }) as const; + + const assertNoNewOverlays = (fixed: typeof edl) => { + const before = new Map(edl.overlays.map((o) => [o.id, o])); + for (const overlay of fixed.overlays) { + const original = before.get(overlay.id); + expect(original, `overlay ${overlay.id} was added`).toBeDefined(); + expect(overlay.kind).toBe(original!.kind); + expect(overlay.text).toBe(original!.text); + expect(overlay.durationMs).toBeLessThanOrEqual(original!.durationMs); + } + expect(fixed.overlays.length).toBeLessThanOrEqual(edl.overlays.length); + }; + + it("swaps an offending clip for a clean alternative segment", () => { + const fix = planAutoFix(edl, report([contentHit(2500, 3000)]), [ + { segmentId: "s2", assetId: "asset-a", startMs: 5000, endMs: 7000 }, + { segmentId: "s9", assetId: "asset-a", startMs: 5500, endMs: 9000 }, + { segmentId: "s8", assetId: "asset-b", startMs: 1000, endMs: 4000 }, + ])!; + expect(fix.changes).toEqual(["clip c2: swapped to segment s8 (content_threshold)"]); + expect(fix.edl.tracks.video.find((c) => c.id === "c2")).toMatchObject({ + assetId: "asset-b", + inMs: 1000, + outMs: 3000, + segmentId: "s8", + atMs: 2000, + }); + expect(edlDurationMs(fix.edl)).toBe(6000); + assertNoNewOverlays(fix.edl); + }); + + it("trims or drops the clip when no alternative fits, and repacks the timeline", () => { + const trimmed = planAutoFix(edl, report([contentHit(3800)]), [])!; + expect(trimmed.changes).toEqual(["clip c2: trimmed the end at 6300 ms (content_threshold)"]); + expect(trimmed.edl.tracks.video.map((c) => [c.id, c.atMs, c.inMs, c.outMs])).toEqual([ + ["c1", 0, 0, 2000], + ["c2", 2000, 5000, 6300], + ["c3", 3300, 8000, 10_000], + ]); + assertNoNewOverlays(trimmed.edl); + expect(trimmed.edl.overlays.find((o) => o.id === "wm")!.durationMs).toBe(5300); + + const dropped = planAutoFix(edl, report([contentHit(2300, 3700)]), [])!; + expect(dropped.changes).toEqual(["clip c2: removed (content_threshold)"]); + expect(dropped.edl.tracks.video.map((c) => c.id)).toEqual(["c1", "c3"]); + expect(dropped.edl.transitions).toEqual([]); + assertNoNewOverlays(dropped.edl); + }); + + it("cuts tracking gaps, widens face redactions, adds a blur for unknown faces, and covers marks", () => { + const fix = planAutoFix( + edl, + report([ + { + check: "anonymization", + severity: "block", + code: "face_not_covered", + message: "", + atMs: 100, + personId: "person-1", + }, + { check: "anonymization", severity: "block", code: "unknown_face", message: "", atMs: 300 }, + { + check: "anonymization", + severity: "block", + code: "mark_visible", + message: "", + atMs: 300, + markId: "mark-1", + }, + { + check: "anonymization", + severity: "block", + code: "tracking_gap", + message: "", + atMs: 4100, + endMs: 4800, + }, + ]), + [], + )!; + expect(fix.edl.redactions.faces).toEqual([ + expect.objectContaining({ id: "f1", paddingPct: 45 }), + expect.objectContaining({ + id: "fix-face-unassigned", + target: { unassignedFaces: true }, + mode: "blur", + }), + ]); + expect(fix.edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "cover" }]); + expect(fix.edl.metadata.aiAlterations).toEqual([]); + expect(fix.edl.tracks.video.find((c) => c.id === "c3")).toMatchObject({ inMs: 9300 }); + assertNoNewOverlays(fix.edl); + }); + + it("stops at the round limit, on passing reports, and when nothing safe applies", () => { + expect(planAutoFix(edl, report([contentHit(2500)], MAX_AUTO_FIX_ROUNDS), [])).toBeNull(); + expect(planAutoFix(edl, report([]), [])).toBeNull(); + const textOnly = report([ + { check: "text", severity: "block", code: "banned_pattern", message: "" }, + ]); + expect(planAutoFix(edl, textOnly, [])).toBeNull(); + const consent = report([ + { check: "consent", severity: "block", code: "consent_missing", message: "", personId: "p" }, + ]); + expect(planAutoFix(edl, consent, [])).toBeNull(); + }); + + it("runs at most three rounds in a loop", () => { + let current = edl; + let rounds = 0; + for (let round = 0; round < 10; round++) { + const fix = planAutoFix(current, report([contentHit(500)], round), []); + if (!fix) break; + current = fix.edl; + rounds += 1; + assertNoNewOverlays(current); + } + expect(rounds).toBe(MAX_AUTO_FIX_ROUNDS); + }); +}); diff --git a/packages/media/src/gate.ts b/packages/media/src/gate.ts new file mode 100644 index 0000000..56d41ea --- /dev/null +++ b/packages/media/src/gate.ts @@ -0,0 +1,699 @@ +import type { + AiAlteration, + ContentCategory, + Edl, + EdlVideoClip, + FaceRedaction, + FfprobeReport, + GateIssue, + GateReport, + NormalizedBox, + PlatformProfile, +} from "@rakazo/contracts"; +import { CONTENT_CATEGORIES, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { + BLENDED_TRANSITIONS, + boxInsideSafeZone, + clipLengthMs, + edlDurationMs, + locateOutputTimeAll, + matchesAspect, + sortedVideoClips, +} from "./edl.js"; + +/** + * Tags a clean export may carry. Everything else is a metadata leak. + * + * - `title`: the only descriptive tag the compiler writes, and only when set. + * - `major_brand`, `minor_version`, `compatible_brands`: the MP4 file-type + * box, always written by the muxer; it names the format, not the author. + * - `handler_name`, `vendor_id`, `language`: per-stream boilerplate the MP4 + * muxer writes ("VideoHandler", "[0][0][0][0]", "und"). + * - `encoder`: the MP4 sample description always names the codec. It is + * allowed only without a version number (bitexact output writes + * "Lavc libx264"); a versioned value such as "Lavf60.16.100" is a leak. + * + * `creation_time`, `location`, `com.apple.quicktime.*`, `make`, `model`, + * `software`, GPS keys and anything unknown always count as leaks. + */ +export const METADATA_ALLOWED_TAGS: readonly string[] = [ + "title", + "major_brand", + "minor_version", + "compatible_brands", + "handler_name", + "vendor_id", + "language", + "encoder", +]; + +const ALLOWED = new Set(METADATA_ALLOWED_TAGS); +/** "Lavf60.16.100", "Lavc60.31", "x264 core 164", "HandBrake 1.7.0". */ +const VERSIONED = /\d+\.\d+|lav[cf]\d|core\s*\d/i; + +/** Tag name without a "format." / "stream.0." style prefix, lower-cased. */ +function tagName(key: string): string { + return key + .trim() + .toLowerCase() + .replace(/^(?:format|streams?(?:[.:/]\d+)?)[.:/]/, ""); +} + +export function findMetadataLeaks(tags: Record): string[] { + return Object.entries(tags) + .filter(([key, value]) => { + const name = tagName(key); + if (!ALLOWED.has(name)) return true; + if (name === "encoder") return VERSIONED.test(value); + return false; + }) + .map(([key]) => key) + .sort(); +} + +export interface GateInput { + profile: PlatformProfile; + edl: Edl; + ffprobe: FfprobeReport; + /** Classifier scores on the rendered output. */ + content: Array<{ tMs: number; scores: Partial> }>; + /** + * Faces re-detected on the rendered output. personId null means the face + * could not be matched to a known person. + */ + faces: Array<{ tMs: number; box: NormalizedBox; personId: string | null; coverage: number }>; + /** Frames where tracking lost a face that must stay anonymized for longer than allowed. */ + trackingGaps: Array<{ startMs: number; endMs: number; personId: string | null }>; + /** Residual visibility 0..1 of treated marks in the output. */ + marks: Array<{ markId: string; tMs: number; residualVisibility: number }>; + /** People visible in the output with their protection and consent state. */ + people: Array<{ + id: string; + anonymizeByDefault: string; + consentRecordedAt: string | null; + ageVerifiedAt: string | null; + }>; + copy: { title: string | null; caption: string | null; tags: string[] }; + aiAlterations: AiAlteration[]; + bundleHasDisclosure: boolean; + watermarkBox: NormalizedBox | null; + sampledFrames: number; + autoFixRound: number; + now: Date; +} + +/** Longest tracking gap the anonymizer may bridge (spec 7). */ +export const MAX_TRACKING_GAP_MS = 500; +/** Residual visibility of a treated mark above which the gate blocks. */ +export const MAX_MARK_RESIDUAL = 0.05; +/** Loudness tolerance around the target, in LU. */ +export const LOUDNESS_TOLERANCE_LU = 2; +const TRUE_PEAK_TOLERANCE_DB = 0.5; + +const ms = (value: number) => Math.max(0, Math.round(value)); + +/** Merge time-sorted hits into ranges when they are at most `gapMs` apart. */ +function ranges(hits: T[], gapMs: number): T[][] { + const sorted = [...hits].sort((a, b) => a.tMs - b.tMs); + const out: T[][] = []; + for (const hit of sorted) { + const last = out.at(-1); + if (last && hit.tMs - last.at(-1)!.tMs <= gapMs) last.push(hit); + else out.push([hit]); + } + return out; +} + +export function checkTechnical( + profile: PlatformProfile, + edl: Edl, + probe: FfprobeReport, +): GateIssue[] { + const issues: GateIssue[] = []; + const block = (code: string, message: string) => + issues.push({ check: "technical", severity: "block", code, message }); + if (probe.width !== edl.output.width || probe.height !== edl.output.height) { + block( + "resolution_mismatch", + `output is ${probe.width}x${probe.height}, plan expects ${edl.output.width}x${edl.output.height}`, + ); + } + if (!matchesAspect(probe.width, probe.height, profile.video.aspectRatios)) { + block("aspect_not_allowed", `${probe.width}x${probe.height} is not an allowed aspect ratio`); + } + if (Math.max(probe.width, probe.height) > Math.max(profile.video.width, profile.video.height)) { + block("resolution_too_high", "output is larger than the profile allows"); + } + if ( + Math.abs(probe.fps - edl.output.fps) > edl.output.fps * 0.01 || + probe.fps > profile.video.fps * 1.001 + ) { + block("fps_mismatch", `output runs at ${probe.fps} fps`); + } + const frameMs = 1000 / Math.max(1, edl.output.fps); + const expected = edlDurationMs(edl); + if (probe.durationMs < profile.duration.hardMinMs - frameMs) { + block("duration_too_short", `${probe.durationMs} ms is below ${profile.duration.hardMinMs} ms`); + } + if (probe.durationMs > profile.duration.hardMaxMs + frameMs) { + block("duration_too_long", `${probe.durationMs} ms is above ${profile.duration.hardMaxMs} ms`); + } + if (Math.abs(probe.durationMs - expected) > Math.max(2 * frameMs, 100)) { + issues.push({ + check: "technical", + severity: "warn", + code: "duration_drift", + message: `output is ${probe.durationMs} ms, plan is ${expected} ms`, + }); + } + if (probe.sizeBytes > profile.video.maxFileSizeMb * 1_000_000) { + block("file_too_large", `file is larger than ${profile.video.maxFileSizeMb} MB`); + } + const codec = probe.videoCodec.toLowerCase(); + const expectedCodec = profile.video.codec === "hevc" ? ["hevc", "h265"] : ["h264", "avc1"]; + if (!expectedCodec.includes(codec)) { + block("codec_mismatch", `video codec ${probe.videoCodec} is not ${profile.video.codec}`); + } + if (probe.audioCodec !== null && probe.audioCodec.toLowerCase() !== "aac") { + issues.push({ + check: "technical", + severity: "warn", + code: "audio_codec", + message: `audio codec ${probe.audioCodec} is not AAC`, + }); + } + const target = edl.targetLufs ?? profile.audio.targetLufs; + if (probe.audioCodec !== null) { + if (probe.integratedLufs === null) { + block("loudness_unmeasured", "loudness was not measured"); + } else if (Math.abs(probe.integratedLufs - target) > LOUDNESS_TOLERANCE_LU) { + block("loudness_off_target", `${probe.integratedLufs} LUFS, target ${target} LUFS`); + } + if ( + probe.truePeakDb !== null && + probe.truePeakDb > profile.audio.truePeakDb + TRUE_PEAK_TOLERANCE_DB + ) { + block("true_peak_exceeded", `true peak ${probe.truePeakDb} dBTP`); + } + } + const leaks = findMetadataLeaks(probe.tags); + if (leaks.length > 0) block("metadata_leak", `metadata not removed: ${leaks.join(", ")}`); + return issues; +} + +const URL_PATTERN = + /\b(?:https?:\/\/|www\.)\S+|\b[a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:com|net|org|io|de|at|ch|eu|ly|co|me|link|shop|store|app|tv|gg|xyz)\b(?:\/\S*)?/i; +const HASHTAG_PATTERN = /#[\p{L}\p{N}_]+/gu; + +/** Profile patterns compile case-insensitively; an invalid one is reported, never thrown. */ +function compilePattern(pattern: string): RegExp | null { + try { + return new RegExp(pattern, "i"); + } catch { + return null; + } +} + +export function checkCopy( + profile: PlatformProfile, + copy: { title: string | null; caption: string | null; tags: string[] }, +): GateIssue[] { + const issues: GateIssue[] = []; + const block = (code: string, message: string) => + issues.push({ check: "text", severity: "block", code, message }); + const { text } = profile; + const title = copy.title ?? ""; + const caption = copy.caption ?? ""; + if (text.titleMaxChars !== undefined && [...title].length > text.titleMaxChars) { + block( + "title_too_long", + `title has ${[...title].length} characters, limit ${text.titleMaxChars}`, + ); + } + if ([...caption].length > text.captionMaxChars) { + block( + "caption_too_long", + `caption has ${[...caption].length} characters, limit ${text.captionMaxChars}`, + ); + } + const hashtags = new Set(); + for (const tag of copy.tags) { + const clean = tag.replace(/^#+/, "").trim().toLowerCase(); + if (clean) hashtags.add(clean); + } + for (const match of caption.matchAll(HASHTAG_PATTERN)) + hashtags.add(match[0].slice(1).toLowerCase()); + if (hashtags.size > text.hashtagLimit) { + block("too_many_hashtags", `${hashtags.size} hashtags, limit ${text.hashtagLimit}`); + } + const fields: Array<[string, string]> = [ + ["title", title], + ["caption", caption], + ...copy.tags.map((tag, i): [string, string] => [`tags[${i}]`, tag]), + ]; + for (const banned of text.bannedPatterns) { + const regex = compilePattern(banned.pattern); + if (!regex) { + block("invalid_banned_pattern", `profile pattern cannot be checked: ${banned.reason}`); + continue; + } + for (const [field, value] of fields) { + if (value && regex.test(value)) block("banned_pattern", `${field}: ${banned.reason}`); + } + } + if (text.linkPolicy !== "allowed") { + for (const [field, value] of fields) { + if (!value || !URL_PATTERN.test(value)) continue; + if (text.linkPolicy === "forbidden") block("link_forbidden", `${field} contains a link`); + else block("link_bio_only", `${field} contains a link; links belong in the bio`); + } + } + return issues; +} + +function checkContent(input: GateInput): GateIssue[] { + const { profile } = input; + const issues: GateIssue[] = []; + const thresholds = profile.content.thresholds; + const gapMs = (1000 / profile.content.sampleFps) * 1.5; + const sampled = Math.max(input.content.length, input.sampledFrames, 1); + for (const category of CONTENT_CATEGORIES) { + const threshold = thresholds[category]; + if (threshold === undefined) continue; + const hits = input.content.flatMap((sample) => { + const score = sample.scores[category]; + return score !== undefined && score > threshold ? [{ tMs: sample.tMs, score }] : []; + }); + if (hits.length === 0) continue; + const blocking = + profile.content.blockOn === "any_frame" || + hits.length / sampled > (profile.content.blockRatio ?? 0); + for (const range of ranges(hits, gapMs)) { + const score = Math.max(...range.map((h) => h.score)); + issues.push({ + check: "content", + severity: blocking ? "block" : "warn", + code: "content_threshold", + message: `${category} ${score.toFixed(2)} above ${threshold}`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + category, + score, + }); + } + } + return issues; +} + +function checkAnonymization(input: GateInput): GateIssue[] { + const issues: GateIssue[] = []; + const protectedIds = new Set( + input.people.filter((p) => p.anonymizeByDefault !== "none").map((p) => p.id), + ); + const gapMs = 1000; + const uncovered = input.faces.filter( + (face) => face.personId !== null && protectedIds.has(face.personId) && face.coverage < 1, + ); + const byPerson = new Map(); + for (const face of uncovered) { + const list = byPerson.get(face.personId!) ?? []; + list.push(face); + byPerson.set(face.personId!, list); + } + for (const [personId, faces] of [...byPerson.entries()].sort((a, b) => + a[0].localeCompare(b[0]), + )) { + for (const range of ranges(faces, gapMs)) { + const coverage = Math.min(...range.map((f) => f.coverage)); + issues.push({ + check: "anonymization", + severity: "block", + code: "face_not_covered", + message: `face covered ${(coverage * 100).toFixed(0)} %`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + personId, + score: coverage, + }); + } + } + if (protectedIds.size > 0) { + const unknown = input.faces.filter((face) => face.personId === null); + for (const range of ranges(unknown, gapMs)) { + issues.push({ + check: "anonymization", + severity: "block", + code: "unknown_face", + message: "a face that cannot be matched appears next to a protected person", + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + }); + } + } + for (const gap of input.trackingGaps) { + if (gap.endMs - gap.startMs <= MAX_TRACKING_GAP_MS) continue; + issues.push({ + check: "anonymization", + severity: "block", + code: "tracking_gap", + message: `face tracking lost for ${Math.round(gap.endMs - gap.startMs)} ms (needs review)`, + atMs: ms(gap.startMs), + endMs: ms(gap.endMs), + ...(gap.personId ? { personId: gap.personId } : {}), + }); + } + const visibleMarks = input.marks.filter((mark) => mark.residualVisibility > MAX_MARK_RESIDUAL); + const byMark = new Map(); + for (const mark of visibleMarks) { + const list = byMark.get(mark.markId) ?? []; + list.push(mark); + byMark.set(mark.markId, list); + } + for (const [markId, marks] of [...byMark.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + for (const range of ranges(marks, gapMs)) { + const residual = Math.max(...range.map((m) => m.residualVisibility)); + issues.push({ + check: "anonymization", + severity: "block", + code: "mark_visible", + message: `treated mark still ${(residual * 100).toFixed(0)} % visible`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + markId, + score: residual, + }); + } + } + return issues; +} + +function checkConsent(input: GateInput): GateIssue[] { + const { consent } = input.profile; + if (!consent.requirePerformerConsent && !consent.requireAgeVerification) return []; + const issues: GateIssue[] = []; + for (const person of [...input.people].sort((a, b) => a.id.localeCompare(b.id))) { + if (consent.requirePerformerConsent && !person.consentRecordedAt) { + issues.push({ + check: "consent", + severity: "block", + code: "consent_missing", + message: "no recorded consent", + personId: person.id, + }); + } + if (consent.requireAgeVerification && !person.ageVerifiedAt) { + issues.push({ + check: "consent", + severity: "block", + code: "age_unverified", + message: "age is not verified", + personId: person.id, + }); + } + } + for (const range of ranges( + input.faces.filter((face) => face.personId === null), + 1000, + )) { + issues.push({ + check: "consent", + severity: "block", + code: "unknown_performer", + message: "a visible face is not assigned to a person with consent", + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + }); + } + return issues; +} + +/** Deterministic gate (spec 6). Any blocking issue fails the report. */ +export function evaluateGate(input: GateInput): GateReport { + const { profile } = input; + const issues: GateIssue[] = [ + ...checkTechnical(profile, input.edl, input.ffprobe), + ...checkContent(input), + ...checkAnonymization(input), + ...checkConsent(input), + ...checkCopy(profile, input.copy), + ]; + const alterations = new Set([...input.aiAlterations, ...input.edl.metadata.aiAlterations]); + if ( + alterations.size > 0 && + profile.disclosure.aiAlteredLabelRequired && + !input.bundleHasDisclosure + ) { + issues.push({ + check: "disclosure", + severity: "block", + code: "ai_label_missing", + message: `AI alterations (${[...alterations].sort().join(", ")}) need the platform label`, + }); + } + if (input.watermarkBox && !boxInsideSafeZone(input.watermarkBox, profile)) { + issues.push({ + check: "brand", + severity: "warn", + code: "watermark_outside_safe_zone", + message: "watermark is covered by platform interface", + }); + } + return { + profileId: profile.id, + passed: !issues.some((issue) => issue.severity === "block"), + checkedAt: input.now.toISOString(), + sampledFrames: Math.max(0, Math.round(input.sampledFrames)), + issues, + autoFixRound: Math.min(MAX_AUTO_FIX_ROUNDS, Math.max(0, Math.round(input.autoFixRound))), + }; +} + +export interface AutoFixResult { + edl: Edl; + changes: string[]; +} + +const PADDING_STEP = 25; +const CUT_MARGIN_MS = 500; +const MIN_KEEP_MS = 500; + +/** Lay clips out back to back again, keeping blended transitions' overlap. */ +function repack(edl: Edl): Edl { + const clips = sortedVideoClips(edl); + const ids = new Set(clips.map((c) => c.id)); + const transitions = edl.transitions.filter((t) => ids.has(t.fromClipId) && ids.has(t.toClipId)); + const adjacent = new Set(); + for (let i = 0; i + 1 < clips.length; i++) adjacent.add(`${clips[i]!.id}>${clips[i + 1]!.id}`); + const kept = transitions.filter((t) => adjacent.has(`${t.fromClipId}>${t.toClipId}`)); + const blend = new Map( + kept + .filter((t) => BLENDED_TRANSITIONS.has(t.kind) && t.durationMs > 0) + .map((t) => [`${t.fromClipId}>${t.toClipId}`, t.durationMs]), + ); + let cursor = 0; + const packed: EdlVideoClip[] = clips.map((clip, i) => { + const previous = clips[i - 1]; + let overlap = previous ? (blend.get(`${previous.id}>${clip.id}`) ?? 0) : 0; + overlap = Math.min(overlap, clipLengthMs(clip) - 1, cursor); + const atMs = Math.max(0, cursor - Math.max(0, overlap)); + cursor = atMs + clipLengthMs(clip); + return { ...clip, atMs }; + }); + const regions = edl.redactions.regions.filter((r) => ids.has(r.clipId)); + const next = { + ...edl, + tracks: { ...edl.tracks, video: packed }, + transitions: kept.filter((t) => { + if (!BLENDED_TRANSITIONS.has(t.kind)) return true; + const from = packed.find((c) => c.id === t.fromClipId)!; + const to = packed.find((c) => c.id === t.toClipId)!; + return t.durationMs < Math.min(clipLengthMs(from), clipLengthMs(to)); + }), + redactions: { ...edl.redactions, regions }, + }; + // Overlays never grow or move; they are only clamped to the new end. + const total = edlDurationMs(next); + next.overlays = edl.overlays.flatMap((overlay) => { + if (overlay.atMs >= total) return []; + return [{ ...overlay, durationMs: Math.min(overlay.durationMs, total - overlay.atMs) }]; + }); + return next; +} + +/** + * Compliance-only repairs: trim or drop offending clips, change crop, widen + * redaction padding, swap to another segment. Never adds obfuscation aimed at + * classifiers. Returns null when the round limit is reached or nothing safe applies. + * + * - Content blocks and tracking gaps: the clip showing that moment is swapped + * for an alternative segment of at least the same length that does not + * overlap the offending source range, else trimmed (when the range sits at + * one end) or dropped. The timeline is then repacked. + * - Uncovered faces: padding of the person's face redaction grows by 25 + * points; a missing redaction is added as blur. + * - Unknown faces: the unassigned-face redaction grows or is added as blur. + * - Visible marks: the mark treatment falls back to cover. + */ +export function planAutoFix( + edl: Edl, + report: GateReport, + alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>, +): AutoFixResult | null { + if (report.autoFixRound >= MAX_AUTO_FIX_ROUNDS || report.passed) return null; + const blocks = report.issues.filter((issue) => issue.severity === "block"); + const changes: string[] = []; + let clips: EdlVideoClip[] = edl.tracks.video.map((clip) => ({ ...clip })); + let faces: FaceRedaction[] = edl.redactions.faces.map((f) => ({ ...f })); + let marks = edl.redactions.marks.map((m) => ({ ...m })); + let clipsChanged = false; + + // Source ranges to remove, per clip. + const offending = new Map>(); + for (const issue of blocks) { + const timeline = + issue.check === "content" || + issue.code === "tracking_gap" || + (issue.code === "mark_visible" && + !edl.redactions.marks.some((m) => m.markId === issue.markId)); + if (!timeline || issue.atMs === undefined) continue; + const endMs = issue.endMs ?? issue.atMs; + const step = Math.max(1, Math.floor(1000 / edl.output.fps)); + const times: number[] = []; + for (let t = issue.atMs; t < endMs; t += step) times.push(t); + times.push(endMs); + for (const t of times) { + for (const hit of locateOutputTimeAll(edl, t)) { + const list = offending.get(hit.clipId) ?? []; + const last = list.at(-1); + if (last && last.reason === issue.code && hit.sourceMs - last.end <= step * 4) { + last.end = Math.max(last.end, hit.sourceMs); + } else { + list.push({ start: hit.sourceMs, end: hit.sourceMs, reason: issue.code }); + } + offending.set(hit.clipId, list); + } + } + } + const usedAlternatives = new Set(); + for (const [clipId, found] of [...offending.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + const clip = clips.find((c) => c.id === clipId); + if (!clip) continue; + const start = Math.max(clip.inMs, Math.min(...found.map((r) => r.start)) - CUT_MARGIN_MS); + const end = Math.min(clip.outMs, Math.max(...found.map((r) => r.end)) + CUT_MARGIN_MS); + const length = clip.outMs - clip.inMs; + const alternative = alternatives + .filter( + (a) => + !usedAlternatives.has(a.segmentId) && + a.segmentId !== clip.segmentId && + a.endMs - a.startMs >= length && + !(a.assetId === clip.assetId && a.startMs < end && start < a.endMs), + ) + .sort((a, b) => a.segmentId.localeCompare(b.segmentId))[0]; + clipsChanged = true; + if (alternative) { + usedAlternatives.add(alternative.segmentId); + Object.assign(clip, { + assetId: alternative.assetId, + segmentId: alternative.segmentId, + inMs: alternative.startMs, + outMs: alternative.startMs + length, + reframe: clip.reframe === "none" ? "none" : "auto", + }); + changes.push( + `clip ${clip.id}: swapped to segment ${alternative.segmentId} (${found[0]!.reason})`, + ); + } else if (start - clip.inMs >= MIN_KEEP_MS && end >= clip.outMs - 1) { + clip.outMs = start; + changes.push(`clip ${clip.id}: trimmed the end at ${start} ms (${found[0]!.reason})`); + } else if (clip.outMs - end >= MIN_KEEP_MS && start <= clip.inMs + 1) { + clip.inMs = end; + changes.push(`clip ${clip.id}: trimmed the start to ${end} ms (${found[0]!.reason})`); + } else { + clips = clips.filter((c) => c.id !== clip.id); + changes.push(`clip ${clip.id}: removed (${found[0]!.reason})`); + } + } + for (const clip of clips) { + if (Array.isArray(clip.reframe)) { + const inside = clip.reframe.filter((k) => k.tMs >= clip.inMs && k.tMs <= clip.outMs); + clip.reframe = inside.length > 0 ? inside : "auto"; + } + } + + const widen = (match: (f: FaceRedaction) => boolean, add: () => FaceRedaction, label: string) => { + const existing = faces.filter(match); + if (existing.length === 0) { + faces = [...faces, add()]; + changes.push(`${label}: added blur redaction`); + return; + } + for (const face of existing) { + if (face.paddingPct >= 100) continue; + face.paddingPct = Math.min(100, face.paddingPct + PADDING_STEP); + changes.push(`${label}: redaction ${face.id} padding ${face.paddingPct} %`); + } + }; + const widened = new Set(); + for (const issue of blocks) { + if (issue.code === "face_not_covered" && issue.personId && !widened.has(issue.personId)) { + const personId = issue.personId; + widened.add(personId); + widen( + (f) => "personId" in f.target && f.target.personId === personId, + () => ({ + id: `fix-face-${widened.size}`, + target: { personId }, + mode: "blur", + paddingPct: 20 + PADDING_STEP, + featherPct: 10, + minSizePct: 2, + }), + `person ${personId}`, + ); + } + if (issue.code === "unknown_face" && !widened.has("(unassigned)")) { + widened.add("(unassigned)"); + widen( + (f) => "unassignedFaces" in f.target, + () => ({ + id: "fix-face-unassigned", + target: { unassignedFaces: true }, + mode: "blur", + paddingPct: 20 + PADDING_STEP, + featherPct: 10, + minSizePct: 2, + }), + "unassigned faces", + ); + } + if (issue.code === "mark_visible" && issue.markId) { + const markId = issue.markId; + marks = marks.map((mark) => { + if (mark.markId !== markId || mark.treatment === "cover") return mark; + changes.push(`mark ${markId}: ${mark.treatment} replaced by cover`); + return { ...mark, treatment: "cover" as const }; + }); + } + } + if (changes.length === 0 || clips.length === 0) return null; + + let next: Edl = { + ...edl, + tracks: { ...edl.tracks, video: clips }, + redactions: { ...edl.redactions, faces, marks }, + }; + if (clipsChanged) next = repack(next); + const stillAltered = new Set( + next.metadata.aiAlterations.filter( + (a) => a !== "inpaint" || marks.some((m) => m.treatment === "inpaint"), + ), + ); + const explicit = edl.metadata.aiAltered && edl.metadata.aiAlterations.length === 0; + next.metadata = { + ...next.metadata, + aiAlterations: [...stillAltered], + aiAltered: stillAltered.size > 0 || explicit, + }; + return { edl: next, changes }; +} diff --git a/packages/media/src/geo.ts b/packages/media/src/geo.ts new file mode 100644 index 0000000..0cda7a6 --- /dev/null +++ b/packages/media/src/geo.ts @@ -0,0 +1,65 @@ +const EARTH_RADIUS_KM = 6371.0088; + +function toRadians(degrees: number): number { + return (degrees * Math.PI) / 180; +} + +/** Great-circle distance in kilometres. */ +export function haversineKm( + a: { lat: number; lon: number }, + b: { lat: number; lon: number }, +): number { + const dLat = toRadians(b.lat - a.lat); + const dLon = toRadians(b.lon - a.lon); + const h = + Math.sin(dLat / 2) ** 2 + + Math.cos(toRadians(a.lat)) * Math.cos(toRadians(b.lat)) * Math.sin(dLon / 2) ** 2; + return 2 * EARTH_RADIUS_KM * Math.asin(Math.min(1, Math.sqrt(h))); +} + +/** Round a coordinate to two decimals (about 1 km) before storage or display. */ +export function roundCoordinate(value: number): number { + return Math.round(value * 100) / 100; +} + +/** Cosine similarity of two equal-length vectors; 0 when either is all zeros. */ +export function cosineSimilarity(a: ArrayLike, b: ArrayLike): number { + if (a.length !== b.length) throw new Error("vector length mismatch"); + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < a.length; i++) { + const x = a[i] as number; + const y = b[i] as number; + dot += x * y; + normA += x * x; + normB += y * y; + } + if (normA === 0 || normB === 0) return 0; + return dot / Math.sqrt(normA * normB); +} + +/** Encode Float32 vectors as little-endian bytes for MediaAnalysis.vectors and centroids. */ +export function encodeVectors(vectors: ArrayLike[]): Uint8Array { + const dim = vectors[0]?.length ?? 0; + const bytes = new Uint8Array(vectors.length * dim * 4); + const view = new DataView(bytes.buffer); + vectors.forEach((vector, row) => { + if (vector.length !== dim) throw new Error("vector length mismatch"); + for (let i = 0; i < dim; i++) view.setFloat32((row * dim + i) * 4, vector[i] as number, true); + }); + return bytes; +} + +export function decodeVectors(bytes: Uint8Array, dim: number): Float32Array[] { + if (dim <= 0 || bytes.byteLength % (dim * 4) !== 0) throw new Error("invalid vector bytes"); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const rows = bytes.byteLength / (dim * 4); + const out: Float32Array[] = []; + for (let row = 0; row < rows; row++) { + const vector = new Float32Array(dim); + for (let i = 0; i < dim; i++) vector[i] = view.getFloat32((row * dim + i) * 4, true); + out.push(vector); + } + return out; +} diff --git a/packages/media/src/index.ts b/packages/media/src/index.ts new file mode 100644 index 0000000..c1111ff --- /dev/null +++ b/packages/media/src/index.ts @@ -0,0 +1,8 @@ +export * from "./clustering.js"; +export * from "./compile.js"; +export * from "./edl.js"; +export * from "./gate.js"; +export * from "./geo.js"; +export * from "./marketing.js"; +export * from "./profiles.js"; +export * from "./teaser.js"; diff --git a/packages/media/src/marketing.test.ts b/packages/media/src/marketing.test.ts new file mode 100644 index 0000000..f2d43fc --- /dev/null +++ b/packages/media/src/marketing.test.ts @@ -0,0 +1,320 @@ +import type { MetricCsvMapping, MotionTemplateInfo } from "@rakazo/contracts"; +import { MetricCsvMappingSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { + buildTrackingUrl, + contrastRatio, + introFits, + normalizeHashtags, + parseCsvRecords, + parseMetricsCsv, + parseMoneyToCents, + proposeIntroVariants, + uploadChecklist, + validateBrandKit, +} from "./marketing.js"; +import { testKit, testPaidProfile, testProfile } from "./test-fixtures.js"; + +const utm = { source: "example-short-vertical", medium: "short", campaign: "spring launch" }; + +describe("buildTrackingUrl", () => { + it("appends UTM parameters and keeps the existing query and fragment", () => { + expect(buildTrackingUrl("https://example.com/shop", utm)).toBe( + "https://example.com/shop?utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch", + ); + expect( + buildTrackingUrl("https://example.com/p?ref=a%20b&x=1&utm_source=old#top", { + ...utm, + content: "thumb-2", + term: "ä&b", + }), + ).toBe( + "https://example.com/p?ref=a%20b&x=1&utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch&utm_content=thumb-2&utm_term=%C3%A4%26b#top", + ); + expect(buildTrackingUrl("http://example.com", utm)).toMatch( + /^http:\/\/example\.com\/\?utm_source=/, + ); + }); + + it("rejects non-http schemes and invalid URLs", () => { + expect(() => buildTrackingUrl("javascript:alert(1)", utm)).toThrow(/http/); + expect(() => buildTrackingUrl("ftp://example.com", utm)).toThrow(/http/); + expect(() => buildTrackingUrl("not a url", utm)).toThrow(/valid/); + }); +}); + +describe("normalizeHashtags", () => { + it("normalizes, dedupes and limits", () => { + expect( + normalizeHashtags(["#Summer", "summer", "##new day", "über-cool!", "", "#"], 10), + ).toEqual(["#Summer", "#newday", "#übercool"]); + expect(normalizeHashtags(["a", "b", "c"], 2)).toEqual(["#a", "#b"]); + expect(normalizeHashtags(["a"], 0)).toEqual([]); + }); +}); + +describe("CSV", () => { + const mapping = (overrides: Partial = {}): MetricCsvMapping => + MetricCsvMappingSchema.parse({ + name: "Example", + columns: { + publication: "Link", + observedAt: "Date", + views: "Views", + clicks: "Clicks", + conversions: "Sales", + revenue: "Revenue", + }, + publicationKey: "tracking_url", + ...overrides, + }); + + it("parses RFC 4180 records with quotes, delimiters and newlines in fields", () => { + expect(parseCsvRecords('a,"b,c","say ""hi""\nthere"\r\n1,2,3\n', ",")).toEqual([ + ["a", "b,c", 'say "hi"\nthere'], + ["1", "2", "3"], + ]); + expect(parseCsvRecords("x;y\n", ";")).toEqual([["x", "y"]]); + expect(parseCsvRecords("", ",")).toEqual([]); + }); + + it("imports metrics with BOM, comma delimiter and major-unit revenue", () => { + const csv = + 'Link,Date,Views,Clicks,Sales,Revenue\n"https://example.com/?utm_content=a,b",2026-09-01,"1,200",30,2,"19.98"\n'; + const result = parseMetricsCsv(csv, mapping()); + expect(result.skipped).toEqual([]); + expect(result.rows).toEqual([ + { + row: 2, + publicationKey: "https://example.com/?utm_content=a,b", + observedAt: new Date("2026-09-01T00:00:00Z"), + views: 1200, + clicks: 30, + conversions: 2, + revenueCents: 1998, + currency: "EUR", + }, + ]); + }); + + it("handles semicolons, tabs, decimal commas, currencies and skipped rows", () => { + const semicolon = [ + "link;date;views;clicks;sales;revenue;currency", + "pub-1;01.09.2026;1.500;12;1;1.234,50 €;", + "pub-2;2026/09/02;10;1;0;12,5;usd", + "pub-3;31.02.2026;1;0;0;0;", + "pub-4;2026-09-03;ten;0;0;0;", + ";2026-09-03;1;0;0;0;", + "pub-5;2026-09-04;1;0;0;-5;", + "pub-6;2026-09-04;1;0;0;1,2,3;", + "pub-7;2026-09-04;1;0;0;5;EURO", + ";;;;;;", + "pub-8;2026-09-05T10:30:00Z;7;;;$3.10;", + ].join("\r\n"); + const result = parseMetricsCsv( + semicolon, + mapping({ + delimiter: ";", + columns: { + publication: "LINK", + observedAt: "date", + views: "views", + clicks: "clicks", + conversions: "sales", + revenue: "revenue", + currency: "currency", + }, + }), + ); + expect(result.rows.map((r) => [r.publicationKey, r.views, r.revenueCents, r.currency])).toEqual( + [ + ["pub-1", 1500, 123_450, "EUR"], + ["pub-2", 10, 1250, "USD"], + ["pub-8", 7, 310, "USD"], + ], + ); + expect(result.rows[2]!.observedAt).toEqual(new Date("2026-09-05T10:30:00Z")); + expect(result.skipped).toEqual([ + { row: 4, reason: "date is not recognized" }, + { row: 5, reason: "views is not a whole number" }, + { row: 6, reason: "publication is empty" }, + { row: 7, reason: "revenue is negative" }, + { row: 8, reason: "revenue is not a number" }, + { row: 9, reason: "currency is not a 3-letter code" }, + ]); + + const tabs = parseMetricsCsv( + "Link\tDate\tRevenue\nx\t2026-09-01\t250\n", + mapping({ + delimiter: "\t", + revenueUnit: "cents", + columns: { publication: "Link", observedAt: "Date", revenue: "Revenue" }, + }), + ); + expect(tabs.rows[0]).toMatchObject({ views: 0, clicks: 0, conversions: 0, revenueCents: 250 }); + }); + + it("reports missing columns and empty files", () => { + expect(parseMetricsCsv("Other,Date\n", mapping())).toEqual({ + rows: [], + skipped: [{ row: 1, reason: "missing columns: Link, Views, Clicks, Sales, Revenue" }], + }); + expect(parseMetricsCsv("", mapping()).skipped).toEqual([{ row: 1, reason: "file is empty" }]); + }); + + it("converts money in either notation to cents", () => { + expect(parseMoneyToCents("1,234.56")).toEqual({ cents: 123_456, currency: null }); + expect(parseMoneyToCents("1.234,56")).toEqual({ cents: 123_456, currency: null }); + expect(parseMoneyToCents("1,234")).toEqual({ cents: 123_400, currency: null }); + expect(parseMoneyToCents("0.995")).toEqual({ cents: 100, currency: null }); + expect(parseMoneyToCents("2.5")).toEqual({ cents: 250, currency: null }); + expect(parseMoneyToCents("CHF 7")).toEqual({ cents: 700, currency: "CHF" }); + expect(parseMoneyToCents("12.345.678,9")).toEqual({ cents: 1_234_567_890, currency: null }); + expect(parseMoneyToCents("1,5,0")).toBeNull(); + }); +}); + +describe("intro", () => { + const templates: MotionTemplateInfo[] = [ + { + id: "intro-glitch", + kind: "intro", + name: "Glitch energy", + minDurationMs: 1000, + maxDurationMs: 5000, + params: [{ name: "primaryColor", type: "color", required: true }], + }, + { + id: "intro-elegant", + kind: "intro", + name: "Elegant fade", + minDurationMs: 2000, + maxDurationMs: 3000, + params: [ + { name: "brandName", type: "string", required: true }, + { name: "sound", type: "string", required: false }, + ], + }, + { + id: "intro-kinetic", + kind: "intro", + name: "Kinetic type", + minDurationMs: 1500, + maxDurationMs: 4000, + params: [], + }, + { + id: "intro-logo", + kind: "intro", + name: "Minimal logo reveal", + minDurationMs: 1500, + maxDurationMs: 4000, + params: [{ name: "logoArtifactId", type: "string", required: true }], + }, + { + id: "outro-a", + kind: "outro", + name: "Outro", + minDurationMs: 1000, + maxDurationMs: 3000, + params: [], + }, + ]; + + it("keeps shorts hook-first and allows short intros on long-form profiles", () => { + expect(introFits(testProfile(), 1500)).toBe(false); + expect(introFits(testProfile(), 0)).toBe(true); + expect(introFits(testPaidProfile(), 3000)).toBe(true); + expect(introFits(testPaidProfile(), 5000)).toBe(false); + }); + + it("proposes three distinct variants ordered by tone", () => { + const kit = testKit({ tone: "calm, premium", logoArtifactIds: ["logo-1"] }); + const variants = proposeIntroVariants(kit, templates, 3); + expect(variants.map((v) => v.template.templateId)).toEqual([ + "intro-elegant", + "intro-logo", + "intro-glitch", + ]); + expect(new Set(variants.map((v) => v.template.templateId)).size).toBe(3); + for (const variant of variants) { + expect(variant.template.durationMs).toBeGreaterThanOrEqual(1500); + expect(variant.template.durationMs).toBeLessThanOrEqual(4000); + } + expect(variants[0]!.template.params).toEqual({ brandName: "Studio Example", sound: "music-1" }); + expect(variants[0]!.rationale).toContain('matches the tone "calm, premium"'); + expect(variants[1]!.template.params).toEqual({ logoArtifactId: "logo-1" }); + expect(proposeIntroVariants(kit, templates, 3)).toEqual(variants); + const energetic = proposeIntroVariants(testKit({ tone: "bold and energetic" }), templates, 3); + expect(energetic[0]!.template).toMatchObject({ + templateId: "intro-glitch", + params: { primaryColor: "#112233" }, + }); + expect(proposeIntroVariants(kit, [], 3)).toEqual([]); + expect( + proposeIntroVariants(kit, templates.slice(0, 1), 3).map((v) => v.template.durationMs), + ).toEqual([2500, 3000, 1500]); + }); +}); + +describe("validateBrandKit", () => { + it("flags low contrast, duplicate presets, unlicensed music and fonts", () => { + expect(validateBrandKit(testKit())).toEqual([]); + expect(contrastRatio("#FFFFFF", "#000000")).toBeCloseTo(21, 5); + const preset = testKit().captionPresets[0]!; + const warnings = validateBrandKit( + testKit({ + colors: { ...testKit().colors, text: "#777777", background: "#888888" }, + captionPresets: [preset, { ...preset, backgroundColor: "#EEEEEE" }], + musicBeds: [ + { artifactId: "m", title: "Borrowed", license: "unknown", commercialUse: false }, + ], + fonts: [{ family: "Nameless", artifactId: "f", role: "body" }], + }), + ); + expect(warnings).toEqual([ + "text on background has contrast 1.3:1, below 4.5:1", + 'caption preset id "clean" is used twice', + 'caption preset "clean" has contrast 1.2:1, below 4.5:1', + 'music bed "Borrowed" is not licensed for commercial use', + 'font "Nameless" has no license note', + ]); + }); +}); + +describe("uploadChecklist", () => { + it("lists the AI label, age restriction, link policy and captions where they apply", () => { + const paid = uploadChecklist({ + profile: testPaidProfile(), + aiAltered: true, + hasCaptions: true, + trackingUrl: "https://example.com/?utm_source=x", + }); + expect(paid).toContain("# Upload checklist: Example paid video"); + expect(paid).toContain("AI-generated or altered content label"); + expect(paid).toContain("age-restricted"); + expect(paid).toContain("Add the tracking link: "); + expect(paid).toContain("SRT or VTT"); + + const short = uploadChecklist({ + profile: testProfile({ captions: { recommended: true, required: true } }), + aiAltered: false, + hasCaptions: false, + trackingUrl: "https://example.com/", + }); + expect(short).not.toContain("AI-generated"); + expect(short).not.toContain("age-restricted"); + expect(short).toContain("in the profile bio, not in the caption"); + expect(short).toContain("Captions are required"); + expect(short).toContain("at most 3 hashtags"); + expect(short.split("\n").filter((l) => l.startsWith("- [ ] "))).toHaveLength(4); + + const relaxed = uploadChecklist({ + profile: testProfile({ disclosure: { aiAlteredLabelRequired: false } }), + aiAltered: true, + hasCaptions: true, + trackingUrl: null, + }); + expect(relaxed).not.toContain("AI-generated"); + }); +}); diff --git a/packages/media/src/marketing.ts b/packages/media/src/marketing.ts new file mode 100644 index 0000000..2a0ee0c --- /dev/null +++ b/packages/media/src/marketing.ts @@ -0,0 +1,532 @@ +import type { + BrandKit, + MetricCsvMapping, + MotionTemplateInfo, + MotionTemplateRef, + PlatformProfile, + Utm, +} from "@rakazo/contracts"; + +const UTM_KEYS: Array<[keyof Utm, string]> = [ + ["source", "utm_source"], + ["medium", "utm_medium"], + ["campaign", "utm_campaign"], + ["content", "utm_content"], + ["term", "utm_term"], +]; + +/** + * Append UTM parameters to an http(s) target. Rejects other schemes. Existing + * query parameters keep their exact encoding; existing utm_* values for the + * keys being set are replaced, and the fragment stays last. + */ +export function buildTrackingUrl(targetUrl: string, utm: Utm): string { + let url: URL; + try { + url = new URL(targetUrl.trim()); + } catch { + throw new Error("tracking target is not a valid URL"); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("tracking target must use http or https"); + } + const setting = UTM_KEYS.filter(([key]) => utm[key]?.trim()); + const replaced = new Set(setting.map(([, name]) => name)); + const kept = url.search + .replace(/^\?/, "") + .split("&") + .filter((part) => { + if (!part) return false; + const rawKey = part.split("=")[0] ?? ""; + let key = rawKey; + try { + key = decodeURIComponent(rawKey.replace(/\+/g, " ")); + } catch { + // keep undecodable keys as they are + } + return !replaced.has(key); + }); + const added = setting.map( + ([key, name]) => `${name}=${encodeURIComponent((utm[key] as string).trim())}`, + ); + const query = [...kept, ...added].join("&"); + return `${url.origin}${url.pathname}${query ? `?${query}` : ""}${url.hash}`; +} + +/** + * Normalize to "#tag": strip leading #, drop spaces and punctuation (letters, + * digits and _ stay, in any script), dedupe case-insensitively, and cap. + */ +export function normalizeHashtags(tags: string[], limit: number): string[] { + const seen = new Set(); + const out: string[] = []; + for (const raw of tags) { + const tag = raw.normalize("NFC").replace(/[^\p{L}\p{N}_]/gu, ""); + if (!tag) continue; + const key = tag.toLocaleLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + out.push(`#${tag}`); + if (out.length >= Math.max(0, limit)) break; + } + return out.slice(0, Math.max(0, limit)); +} + +export interface ParsedMetricRow { + row: number; + publicationKey: string; + observedAt: Date; + views: number; + clicks: number; + conversions: number; + revenueCents: number; + currency: string; +} + +/** RFC 4180 records; quoted fields may hold delimiters, quotes ("") and newlines. */ +export function parseCsvRecords(csv: string, delimiter: string): string[][] { + const text = csv.replace(/^/, ""); + const records: string[][] = []; + let record: string[] = []; + let field = ""; + let quoted = false; + let i = 0; + while (i < text.length) { + const char = text[i]!; + if (quoted) { + if (char === '"') { + if (text[i + 1] === '"') { + field += '"'; + i += 2; + continue; + } + quoted = false; + i += 1; + continue; + } + field += char; + i += 1; + continue; + } + if (char === '"' && field.length === 0) { + quoted = true; + i += 1; + } else if (char === delimiter) { + record.push(field); + field = ""; + i += 1; + } else if (char === "\r" || char === "\n") { + record.push(field); + records.push(record); + record = []; + field = ""; + i += char === "\r" && text[i + 1] === "\n" ? 2 : 1; + } else { + field += char; + i += 1; + } + } + if (field.length > 0 || record.length > 0) { + record.push(field); + records.push(record); + } + return records; +} + +const CURRENCY_SYMBOLS: Record = { "€": "EUR", $: "USD", "£": "GBP", "¥": "JPY" }; + +/** Whole, non-negative count with optional thousands separators. */ +function parseCount(raw: string): number | null { + const value = raw.trim().replace(/[\s  ']/g, ""); + if (value === "") return 0; + if (/^\d+$/.test(value)) return Number(value); + if (/^\d{1,3}([.,]\d{3})+$/.test(value) && new Set(value.match(/[.,]/g)).size === 1) { + return Number(value.replace(/[.,]/g, "")); + } + return null; +} + +/** + * Money in major units to cents. With both "," and "." the later one is the + * decimal separator; a single separator followed by one or two digits is + * decimal, by exactly three digits after 1–999 it groups thousands ("1,234"), + * otherwise it is decimal ("0.995"). Extra decimals round half up. + */ +export function parseMoneyToCents(raw: string): { cents: number; currency: string | null } | null { + let value = raw.trim(); + let currency: string | null = null; + for (const [symbol, code] of Object.entries(CURRENCY_SYMBOLS)) { + if (value.includes(symbol)) { + currency = code; + value = value.replace(symbol, ""); + } + } + const code = value.match(/[A-Za-z]{3}/)?.[0]; + if (code) { + currency = code.toUpperCase(); + value = value.replace(code, ""); + } + value = value.replace(/[\s  ']/g, ""); + if (value === "") return { cents: 0, currency }; + if (!/^\d[\d.,]*$/.test(value)) return null; + const lastComma = value.lastIndexOf(","); + const lastDot = value.lastIndexOf("."); + let integer = value; + let fraction = ""; + if (lastComma >= 0 && lastDot >= 0) { + const decimal = Math.max(lastComma, lastDot); + integer = value.slice(0, decimal); + fraction = value.slice(decimal + 1); + } else if (lastComma >= 0 || lastDot >= 0) { + const separator = lastComma >= 0 ? "," : "."; + const parts = value.split(separator); + const tail = parts.at(-1)!; + const decimal = + parts.length === 2 && + tail.length >= 1 && + (tail.length <= 2 || !/^[1-9]\d{0,2}$/.test(parts[0]!)); + if (decimal) { + integer = parts[0]!; + fraction = tail; + } else if (parts.slice(1).every((part) => part.length === 3)) { + integer = parts.join(""); + } else { + return null; + } + } + integer = integer.replace(/[.,]/g, ""); + if (!/^\d+$/.test(integer) || !/^\d*$/.test(fraction)) return null; + const twoDigits = `${fraction}00`.slice(0, 2); + const roundUp = fraction.length > 2 && Number(fraction[2]) >= 5 ? 1 : 0; + return { cents: Number(integer) * 100 + Number(twoDigits) + roundUp, currency }; +} + +/** ISO dates and times, YYYY/MM/DD, and DD.MM.YYYY. Ambiguous slash dates are rejected. */ +function parseDate(raw: string): Date | null { + const value = raw.trim(); + let match = value.match(/^(\d{4})[-/](\d{2})[-/](\d{2})$/); + if (match) return utcDate(Number(match[1]), Number(match[2]), Number(match[3])); + match = value.match(/^(\d{1,2})\.(\d{1,2})\.(\d{4})$/); + if (match) return utcDate(Number(match[3]), Number(match[2]), Number(match[1])); + if (/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(value)) { + const date = new Date(value.replace(" ", "T")); + return Number.isNaN(date.getTime()) ? null : date; + } + return null; +} + +function utcDate(year: number, month: number, day: number): Date | null { + const date = new Date(Date.UTC(year, month - 1, day)); + return date.getUTCFullYear() === year && + date.getUTCMonth() === month - 1 && + date.getUTCDate() === day + ? date + : null; +} + +/** RFC 4180 CSV parsing with the saved column mapping. Rows count from the header as row 1. */ +export function parseMetricsCsv( + csv: string, + mapping: MetricCsvMapping, +): { rows: ParsedMetricRow[]; skipped: Array<{ row: number; reason: string }> } { + const records = parseCsvRecords(csv, mapping.delimiter ?? ","); + const header = records[0]; + const rows: ParsedMetricRow[] = []; + const skipped: Array<{ row: number; reason: string }> = []; + if (!header) return { rows, skipped: [{ row: 1, reason: "file is empty" }] }; + const names = header.map((name) => name.trim()); + const column = (name: string | undefined): number => { + if (!name) return -1; + const exact = names.indexOf(name.trim()); + if (exact >= 0) return exact; + const lower = name.trim().toLowerCase(); + return names.findIndex((n) => n.toLowerCase() === lower); + }; + const columns = mapping.columns; + const index = { + publication: column(columns.publication), + observedAt: column(columns.observedAt), + views: column(columns.views), + clicks: column(columns.clicks), + conversions: column(columns.conversions), + revenue: column(columns.revenue), + currency: column(columns.currency), + }; + const missing = (Object.keys(index) as Array).filter( + (key) => columns[key] !== undefined && index[key] < 0, + ); + if (missing.length > 0) { + return { + rows, + skipped: [ + { row: 1, reason: `missing columns: ${missing.map((k) => columns[k]).join(", ")}` }, + ], + }; + } + const defaultCurrency = (mapping.defaultCurrency ?? "EUR").toUpperCase(); + + for (const [offset, record] of records.slice(1).entries()) { + const row = offset + 2; + if (record.every((field) => field.trim() === "")) continue; + const cell = (i: number) => (i >= 0 ? (record[i] ?? "") : ""); + const publicationKey = cell(index.publication).trim(); + if (!publicationKey) { + skipped.push({ row, reason: "publication is empty" }); + continue; + } + const observedAt = parseDate(cell(index.observedAt)); + if (!observedAt) { + skipped.push({ row, reason: "date is not recognized" }); + continue; + } + const counts = { + views: parseCount(cell(index.views)), + clicks: parseCount(cell(index.clicks)), + conversions: parseCount(cell(index.conversions)), + }; + const invalid = (Object.keys(counts) as Array).find( + (key) => counts[key] === null, + ); + if (invalid) { + skipped.push({ row, reason: `${invalid} is not a whole number` }); + continue; + } + let revenueCents = 0; + let currency: string | null = null; + const revenueRaw = cell(index.revenue); + if (revenueRaw.trim()) { + if (/^\s*-/.test(revenueRaw)) { + skipped.push({ row, reason: "revenue is negative" }); + continue; + } + if (mapping.revenueUnit === "cents") { + const value = parseCount(revenueRaw); + if (value === null) { + skipped.push({ row, reason: "revenue is not a whole number" }); + continue; + } + revenueCents = value; + } else { + const money = parseMoneyToCents(revenueRaw); + if (!money) { + skipped.push({ row, reason: "revenue is not a number" }); + continue; + } + revenueCents = money.cents; + currency = money.currency; + } + } + const currencyCell = cell(index.currency).trim().toUpperCase(); + if (currencyCell && !/^[A-Z]{3}$/.test(currencyCell)) { + skipped.push({ row, reason: "currency is not a 3-letter code" }); + continue; + } + rows.push({ + row, + publicationKey, + observedAt, + views: counts.views ?? 0, + clicks: counts.clicks ?? 0, + conversions: counts.conversions ?? 0, + revenueCents, + currency: currencyCell || currency || defaultCurrency, + }); + } + return { rows, skipped }; +} + +const SHORT_FORM_MAX_MS = 180_000; +const MAX_INTRO_MS = 4000; + +/** Vertical profiles capped at three minutes: the hook must come before any branding. */ +export function isShortForm(profile: PlatformProfile): boolean { + return ( + profile.video.aspectRatios.includes("9:16") && profile.duration.hardMaxMs <= SHORT_FORM_MAX_MS + ); +} + +/** + * Whether the brand intro may open the video. Short-form vertical profiles + * always put the hook first (the brand appears as watermark or end card). + * Elsewhere the intro may take at most 4 s and a tenth of the maximum length. + */ +export function introFits(profile: PlatformProfile, introMs: number): boolean { + if (introMs <= 0) return true; + if (isShortForm(profile)) return false; + return introMs <= Math.min(MAX_INTRO_MS, profile.duration.hardMaxMs / 10); +} + +const TONE_STYLES: Array<{ pattern: RegExp; keywords: string[] }> = [ + { + pattern: /energ|bold|dynamic|loud|wild|sport|fast|laut|schnell|kraft/i, + keywords: ["glitch", "energy", "kinetic"], + }, + { + pattern: /elegan|calm|luxur|soft|premium|ruhig|edel|sanft|warm/i, + keywords: ["elegant", "fade", "minimal"], + }, + { + pattern: /minimal|clean|simple|schlicht|klar|modern/i, + keywords: ["minimal", "logo", "fade"], + }, + { pattern: /playful|fun|witzig|verspielt|bunt/i, keywords: ["kinetic", "typo", "glitch"] }, +]; + +const PARAM_SOURCES: Record string | undefined> = { + primarycolor: (kit) => kit.colors.primary, + secondarycolor: (kit) => kit.colors.secondary, + accentcolor: (kit) => kit.colors.accent, + textcolor: (kit) => kit.colors.text, + backgroundcolor: (kit) => kit.colors.background, + brandname: (kit) => kit.name, + title: (kit) => kit.name, + name: (kit) => kit.name, + logoartifactid: (kit) => kit.logoArtifactIds[0], + logo: (kit) => kit.logoArtifactIds[0], + fontfamily: (kit) => (kit.fonts.find((f) => f.role === "heading") ?? kit.fonts[0])?.family, + soundartifactid: (kit) => kit.musicBeds.find((bed) => bed.commercialUse)?.artifactId, + sound: (kit) => kit.musicBeds.find((bed) => bed.commercialUse)?.artifactId, +}; + +const VARIANT_DURATIONS = [2500, 2000, 3000, 3500, 1500, 4000]; + +/** + * Deterministic intro proposals from the brand kit, tone, and available + * templates: intro templates are ranked by how well their id and name match + * the tone's style keywords (then by id), each proposal uses a distinct + * template while there are enough, and durations stay within 1.5–4 s and the + * template's own limits. Parameters are filled from the kit by name. + */ +export function proposeIntroVariants( + kit: BrandKit, + templates: MotionTemplateInfo[], + count: number, +): Array<{ template: MotionTemplateRef; rationale: string }> { + const intros = templates + .filter((t) => t.kind === "intro") + .sort((a, b) => a.id.localeCompare(b.id)); + if (intros.length === 0 || count <= 0) return []; + const tone = `${kit.tone ?? ""} ${kit.audience ?? ""}`; + const keywords = TONE_STYLES.filter((style) => style.pattern.test(tone)).flatMap( + (style) => style.keywords, + ); + const affinity = (template: MotionTemplateInfo) => { + const haystack = `${template.id} ${template.name}`.toLowerCase(); + return keywords.reduce( + (score, keyword, index) => score + (haystack.includes(keyword) ? keywords.length - index : 0), + 0, + ); + }; + const ranked = [...intros].sort((a, b) => affinity(b) - affinity(a) || a.id.localeCompare(b.id)); + + return Array.from({ length: count }, (_, index) => { + const template = ranked[index % ranked.length]!; + const round = Math.floor(index / ranked.length); + const wanted = VARIANT_DURATIONS[(index + round) % VARIANT_DURATIONS.length]!; + const durationMs = Math.min( + Math.max(wanted, 1500, template.minDurationMs), + Math.min(4000, template.maxDurationMs), + ); + const params: Record = {}; + for (const param of template.params) { + const source = PARAM_SOURCES[param.name.toLowerCase().replace(/[^a-z]/g, "")]; + const value = source?.(kit); + if (value !== undefined) params[param.name] = value.slice(0, 200); + } + const matched = affinity(template) > 0 && kit.tone ? ` matches the tone "${kit.tone}"` : ""; + return { + template: { templateId: template.id, params, durationMs: Math.max(0, durationMs) }, + rationale: `${template.name}${matched}, ${(durationMs / 1000).toFixed(1)} s`, + }; + }); +} + +function luminance(hex: string): number { + const channel = (offset: number) => { + const value = Number.parseInt(hex.slice(offset, offset + 2), 16) / 255; + return value <= 0.03928 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4; + }; + return 0.2126 * channel(1) + 0.7152 * channel(3) + 0.0722 * channel(5); +} + +/** WCAG contrast ratio of two #RRGGBB colours. */ +export function contrastRatio(a: string, b: string): number { + const [light, dark] = [luminance(a), luminance(b)].sort((x, y) => y - x) as [number, number]; + return (light + 0.05) / (dark + 0.05); +} + +const MIN_CONTRAST = 4.5; + +/** Structural brand-kit checks beyond the schema (contrast, duplicate presets). */ +export function validateBrandKit(kit: BrandKit): string[] { + const warnings: string[] = []; + const ratio = contrastRatio(kit.colors.text, kit.colors.background); + if (ratio < MIN_CONTRAST) { + warnings.push(`text on background has contrast ${ratio.toFixed(1)}:1, below ${MIN_CONTRAST}:1`); + } + const seen = new Set(); + for (const preset of kit.captionPresets) { + if (seen.has(preset.id)) warnings.push(`caption preset id "${preset.id}" is used twice`); + seen.add(preset.id); + if (preset.backgroundColor) { + const presetRatio = contrastRatio(preset.textColor, preset.backgroundColor); + if (presetRatio < MIN_CONTRAST) { + warnings.push( + `caption preset "${preset.id}" has contrast ${presetRatio.toFixed(1)}:1, below ${MIN_CONTRAST}:1`, + ); + } + } + } + for (const bed of kit.musicBeds) { + if (!bed.commercialUse) + warnings.push(`music bed "${bed.title}" is not licensed for commercial use`); + } + for (const font of kit.fonts) { + if (!font.license?.trim()) warnings.push(`font "${font.family}" has no license note`); + } + if (kit.watermark && !kit.watermark.artifactId) warnings.push("watermark has no image"); + return warnings; +} + +/** Upload checklist markdown for an export bundle. */ +export function uploadChecklist(input: { + profile: PlatformProfile; + aiAltered: boolean; + hasCaptions: boolean; + trackingUrl: string | null; +}): string { + const { profile } = input; + const items: string[] = ["Upload the video file and choose the poster as cover image."]; + if (input.aiAltered && profile.disclosure.aiAlteredLabelRequired) { + items.push( + "Set the platform's AI-generated or altered content label: this video contains AI alterations.", + ); + } + if (profile.disclosure.ageRestrictedFlag) items.push("Mark the upload as age-restricted."); + if (profile.disclosure.paidPartnershipLabel) { + items.push("Turn on the paid partnership label if this post is sponsored."); + } + if (input.trackingUrl) { + const link = `<${input.trackingUrl}>`; + if (profile.text.linkPolicy === "allowed") items.push(`Add the tracking link: ${link}`); + else if (profile.text.linkPolicy === "bio_only") { + items.push(`Put the tracking link in the profile bio, not in the caption: ${link}`); + } else items.push("Do not add links: this platform does not allow them."); + } else if (profile.text.linkPolicy === "forbidden") { + items.push("Do not add links: this platform does not allow them."); + } + if (input.hasCaptions) { + items.push("Upload the SRT or VTT subtitles, or confirm the burned-in captions are readable."); + } else if (profile.captions.required) { + items.push("Captions are required on this platform: add them before publishing."); + } + const limits = [ + profile.text.titleMaxChars ? `title up to ${profile.text.titleMaxChars} characters` : null, + `caption up to ${profile.text.captionMaxChars} characters`, + `at most ${profile.text.hashtagLimit} hashtags`, + ].filter(Boolean); + items.push(`Paste the caption from caption.txt (${limits.join(", ")}).`); + return `# Upload checklist: ${profile.name.replace(/[\r\n]+/g, " ")}\n\n${items + .map((item) => `- [ ] ${item}`) + .join("\n")}\n`; +} diff --git a/packages/media/src/node.ts b/packages/media/src/node.ts new file mode 100644 index 0000000..d316369 --- /dev/null +++ b/packages/media/src/node.ts @@ -0,0 +1,88 @@ +import { lstat, readdir, readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { PlatformProfile } from "@rakazo/contracts"; +import type { ProfileLoadError } from "./profiles.js"; +import { parseProfiles } from "./profiles.js"; + +/** Directory of the profile JSON files shipped with this package. */ +export const BUILTIN_PROFILES_DIR = fileURLToPath(new URL("../profiles/", import.meta.url)); + +/** Read every *.json in a directory; a missing directory yields no profiles. */ +export async function loadProfilesFromDir( + dir: string, +): Promise<{ profiles: PlatformProfile[]; errors: ProfileLoadError[] }> { + let names: string[]; + try { + names = (await readdir(dir)).filter((name) => name.endsWith(".json")).sort(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return { profiles: [], errors: [] }; + throw error; + } + const documents: Array<{ file: string; json: unknown }> = []; + const errors: ProfileLoadError[] = []; + for (const name of names) { + const file = path.join(dir, name); + try { + documents.push({ file: name, json: JSON.parse(await readFile(file, "utf8")) }); + } catch (error) { + errors.push({ file: name, message: error instanceof Error ? error.message : String(error) }); + } + } + const parsed = parseProfiles(documents); + return { profiles: parsed.profiles, errors: [...errors, ...parsed.errors] }; +} + +/** Expand a leading "~/" against the given home directory. */ +export function expandHome(input: string, home: string): string { + return input === "~" ? home : input.startsWith("~/") ? path.join(home, input.slice(2)) : input; +} + +export class PathOutsideRootError extends Error { + constructor(readonly requested: string) { + super("path is outside the allowed root"); + this.name = "PathOutsideRootError"; + } +} + +function isInside(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +/** + * Resolve `relative` inside `root`. Rejects absolute inputs, traversal, NUL + * bytes, and any existing path whose real location (after symlinks) leaves the + * real root. A path that does not exist yet is checked through its nearest + * existing ancestor. + */ +export async function resolveInside(root: string, relative: string): Promise { + if (relative.includes("\0") || path.isAbsolute(relative)) + throw new PathOutsideRootError(relative); + const realRoot = await realpath(root); + const lexical = path.resolve(realRoot, relative); + if (!isInside(realRoot, lexical)) throw new PathOutsideRootError(relative); + let probe = lexical; + for (;;) { + try { + await lstat(probe); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + const parent = path.dirname(probe); + if (parent === probe) throw new PathOutsideRootError(relative); + probe = parent; + } + } + const real = await realpath(probe); + if (!isInside(realRoot, real)) throw new PathOutsideRootError(relative); + return lexical; +} + +/** Like resolveInside for an absolute path that must lie within root. */ +export async function assertInside(root: string, absolute: string): Promise { + if (!path.isAbsolute(absolute)) throw new PathOutsideRootError(absolute); + const realRoot = await realpath(root); + const relative = path.relative(realRoot, path.resolve(absolute)); + return resolveInside(realRoot, relative === "" ? "." : relative); +} diff --git a/packages/media/src/profiles.ts b/packages/media/src/profiles.ts new file mode 100644 index 0000000..3cfb77d --- /dev/null +++ b/packages/media/src/profiles.ts @@ -0,0 +1,78 @@ +import type { PlatformProfile, PlatformProfileSummary } from "@rakazo/contracts"; +import { PlatformProfileSchema } from "@rakazo/contracts"; + +export const PROFILE_STALE_DAYS = 90; + +export interface ProfileLoadError { + file: string; + message: string; +} + +export interface ProfileSet { + profiles: Map; + errors: ProfileLoadError[]; +} + +/** Parse raw JSON documents; invalid ones become errors and are skipped whole. */ +export function parseProfiles(documents: Array<{ file: string; json: unknown }>): { + profiles: PlatformProfile[]; + errors: ProfileLoadError[]; +} { + const profiles: PlatformProfile[] = []; + const errors: ProfileLoadError[] = []; + const seen = new Set(); + for (const document of documents) { + const parsed = PlatformProfileSchema.safeParse(document.json); + if (!parsed.success) { + errors.push({ + file: document.file, + message: parsed.error.issues + .map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + .join("; "), + }); + continue; + } + if (seen.has(parsed.data.id)) { + errors.push({ file: document.file, message: `duplicate profile id ${parsed.data.id}` }); + continue; + } + seen.add(parsed.data.id); + profiles.push(parsed.data); + } + return { profiles, errors }; +} + +/** Private profiles override builtin ones with the same id. */ +export function mergeProfiles( + builtin: PlatformProfile[], + privateProfiles: PlatformProfile[], +): ProfileSet { + const profiles: ProfileSet["profiles"] = new Map(); + for (const profile of builtin) profiles.set(profile.id, { profile, origin: "builtin" }); + for (const profile of privateProfiles) profiles.set(profile.id, { profile, origin: "private" }); + return { profiles, errors: [] }; +} + +export function isProfileStale(profile: PlatformProfile, now: Date): boolean { + const reviewed = Date.parse(`${profile.reviewedAt}T00:00:00Z`); + return now.getTime() - reviewed > PROFILE_STALE_DAYS * 24 * 60 * 60 * 1000; +} + +export function summarizeProfile( + profile: PlatformProfile, + origin: "builtin" | "private", + now: Date, +): PlatformProfileSummary { + return { + id: profile.id, + name: profile.name, + kind: profile.kind, + origin, + unverified: profile.unverified, + stale: isProfileStale(profile, now), + aspectRatios: profile.video.aspectRatios, + hardMinMs: profile.duration.hardMinMs, + hardMaxMs: profile.duration.hardMaxMs, + recommendedMs: profile.duration.recommended, + }; +} diff --git a/packages/media/src/shared.test.ts b/packages/media/src/shared.test.ts new file mode 100644 index 0000000..00ff282 --- /dev/null +++ b/packages/media/src/shared.test.ts @@ -0,0 +1,144 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + cosineSimilarity, + decodeVectors, + encodeVectors, + haversineKm, + roundCoordinate, +} from "./geo.js"; +import { + assertInside, + expandHome, + loadProfilesFromDir, + PathOutsideRootError, + resolveInside, +} from "./node.js"; +import { mergeProfiles, parseProfiles, summarizeProfile } from "./profiles.js"; + +const profileJson = (id: string, overrides: Record = {}) => ({ + id, + name: `Profile ${id}`, + kind: "free_social", + reviewedAt: "2026-09-01", + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 500, + }, + duration: { hardMinMs: 1000, hardMaxMs: 60000, recommended: [15000] }, + safeZones: { topPct: 10, bottomPct: 20, leftPct: 5, rightPct: 10 }, + captions: { recommended: true }, + audio: {}, + text: { captionMaxChars: 2200, hashtagLimit: 5, linkPolicy: "bio_only" }, + content: { thresholds: { explicit_nudity: 0.2 } }, + disclosure: { aiAlteredLabelRequired: true }, + consent: { requirePerformerConsent: false, requireAgeVerification: false }, + export: {}, + ...overrides, +}); + +describe("geo", () => { + it("measures distance and rounds coordinates", () => { + expect(haversineKm({ lat: 52.52, lon: 13.405 }, { lat: 48.137, lon: 11.575 })).toBeCloseTo( + 504, + -1, + ); + expect(haversineKm({ lat: 1, lon: 1 }, { lat: 1, lon: 1 })).toBe(0); + expect(roundCoordinate(52.51678)).toBe(52.52); + }); + + it("computes cosine similarity and round-trips vectors", () => { + expect(cosineSimilarity([1, 0], [1, 0])).toBe(1); + expect(cosineSimilarity([1, 0], [0, 1])).toBe(0); + expect(cosineSimilarity([0, 0], [1, 1])).toBe(0); + expect(() => cosineSimilarity([1], [1, 2])).toThrow(); + const decoded = decodeVectors( + encodeVectors([ + [1, 2.5], + [-3, 0], + ]), + 2, + ); + expect(Array.from(decoded[1] ?? [])).toEqual([-3, 0]); + expect(() => decodeVectors(new Uint8Array(6), 2)).toThrow(); + }); +}); + +describe("profiles", () => { + it("skips invalid and duplicate documents whole", () => { + const result = parseProfiles([ + { file: "a.json", json: profileJson("a") }, + { file: "b.json", json: { id: "b" } }, + { file: "a2.json", json: profileJson("a") }, + { file: "c.json", json: profileJson("c", { duration: { hardMinMs: 9, hardMaxMs: 1 } }) }, + ]); + expect(result.profiles.map((p) => p.id)).toEqual(["a"]); + expect(result.errors.map((e) => e.file)).toEqual(["b.json", "a2.json", "c.json"]); + }); + + it("lets private profiles override builtin ones and flags stale ones", () => { + const builtin = parseProfiles([{ file: "x", json: profileJson("shared") }]).profiles; + const own = parseProfiles([ + { file: "y", json: profileJson("shared", { name: "Mine" }) }, + ]).profiles; + const merged = mergeProfiles(builtin, own); + expect(merged.profiles.get("shared")?.origin).toBe("private"); + expect(merged.profiles.get("shared")?.profile.name).toBe("Mine"); + const summary = summarizeProfile(builtin[0]!, "builtin", new Date("2027-01-15T00:00:00Z")); + expect(summary.stale).toBe(true); + expect(summarizeProfile(builtin[0]!, "builtin", new Date("2026-09-20T00:00:00Z")).stale).toBe( + false, + ); + }); +}); + +describe("node helpers", () => { + let root: string; + let outside: string; + + beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "media-root-")); + outside = await mkdtemp(path.join(tmpdir(), "media-outside-")); + await mkdir(path.join(root, "a")); + await writeFile(path.join(root, "a", "f.txt"), "x"); + await symlink(outside, path.join(root, "escape")); + await writeFile(path.join(root, "good.json"), JSON.stringify(profileJson("good"))); + await writeFile(path.join(root, "broken.json"), "{"); + }); + + afterAll(async () => { + await rm(root, { recursive: true, force: true }); + await rm(outside, { recursive: true, force: true }); + }); + + it("resolves paths inside the root and rejects escapes", async () => { + expect(await resolveInside(root, "a/f.txt")).toMatch(/a\/f\.txt$/); + expect(await resolveInside(root, "a/new/file.mp4")).toMatch(/a\/new\/file\.mp4$/); + await expect(resolveInside(root, "../x")).rejects.toBeInstanceOf(PathOutsideRootError); + await expect(resolveInside(root, "/etc/passwd")).rejects.toBeInstanceOf(PathOutsideRootError); + await expect(resolveInside(root, "escape/secret")).rejects.toBeInstanceOf(PathOutsideRootError); + await expect(resolveInside(root, "a/\0x")).rejects.toBeInstanceOf(PathOutsideRootError); + await expect(assertInside(root, path.join(outside, "x"))).rejects.toBeInstanceOf( + PathOutsideRootError, + ); + expect(await assertInside(root, path.join(root, "a"))).toMatch(/\/a$/); + }); + + it("loads profile directories and reports broken files", async () => { + const loaded = await loadProfilesFromDir(root); + expect(loaded.profiles.map((p) => p.id)).toEqual(["good"]); + expect(loaded.errors.map((e) => e.file)).toEqual(["broken.json"]); + expect(await loadProfilesFromDir(path.join(root, "missing"))).toEqual({ + profiles: [], + errors: [], + }); + expect(expandHome("~/x", "/home/u")).toBe("/home/u/x"); + }); +}); diff --git a/packages/media/src/teaser.test.ts b/packages/media/src/teaser.test.ts new file mode 100644 index 0000000..e2f218c --- /dev/null +++ b/packages/media/src/teaser.test.ts @@ -0,0 +1,334 @@ +import { + MediaProjectGoalSchema, + TEASER_FEATURES, + TEASER_LEARNING_MIN_SAMPLES, +} from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { validateEdl } from "./edl.js"; +import type { ScoredSegmentInput, SegmentFeatures } from "./teaser.js"; +import { + autoPlanEdl, + defaultTeaserWeights, + fitTeaserWeights, + rankThumbnailFrames, + scoreSegment, + selectTeaserSegments, + speechHookScore, +} from "./teaser.js"; +import { testContext, testKit, testPaidProfile, testProfile } from "./test-fixtures.js"; + +function segment( + id: string, + assetId: string, + startMs: number, + endMs: number, + features: SegmentFeatures, + extra: Partial = {}, +): ScoredSegmentInput { + return { + id, + assetId, + startMs, + endMs, + capturedAt: null, + features, + paidOnly: false, + transcript: null, + ...extra, + }; +} + +/** One 30 s asset building up to a climax at 20–24 s, plus a second asset. */ +const segments: ScoredSegmentInput[] = [ + segment("s1", "asset-a", 0, 4000, { motion: 0.2, sharpness: 0.8 }), + segment( + "s2", + "asset-a", + 4000, + 8000, + { motion: 0.3, audioPeak: 0.3 }, + { transcript: "Wait, you won't believe what happens next?" }, + ), + segment("s3", "asset-a", 8000, 12_000, { motion: 0.5, faces: 0.6 }), + segment("s4", "asset-a", 12_000, 16_000, { motion: 0.6, audioPeak: 0.5 }), + segment("s5", "asset-a", 16_000, 20_000, { motion: 0.7, novelty: 0.6 }, { paidOnly: true }), + segment("s6", "asset-a", 20_000, 24_000, { + motion: 1, + audioPeak: 1, + faces: 1, + novelty: 1, + aesthetics: 1, + }), + segment("s7", "asset-a", 24_000, 28_000, { motion: 0.4 }), + segment("s8", "asset-b", 0, 3000, { motion: 0.5, aesthetics: 0.7 }), +]; + +describe("weights and scores", () => { + it("defines every feature for every goal", () => { + for (const goal of MediaProjectGoalSchema.options) { + const weights = defaultTeaserWeights(goal); + expect(Object.keys(weights).sort()).toEqual([...TEASER_FEATURES].sort()); + expect(Object.values(weights).reduce((a, b) => a + b, 0)).toBeCloseTo(1, 6); + } + expect(defaultTeaserWeights("teaser").speechHook).toBeGreaterThan( + defaultTeaserWeights("highlight").speechHook, + ); + }); + + it("scores as a weighted mean of clamped features", () => { + const weights = defaultTeaserWeights("teaser"); + expect(scoreSegment({}, weights)).toBe(0); + const all = Object.fromEntries(TEASER_FEATURES.map((f) => [f, 1])); + expect(scoreSegment(all, weights)).toBeCloseTo(1, 6); + expect(scoreSegment({ motion: 5 }, weights)).toBeCloseTo(weights.motion, 6); + expect(scoreSegment({ motion: Number.NaN }, weights)).toBe(0); + expect(scoreSegment({ motion: 1 }, { ...weights, motion: 0 })).toBe(0); + }); + + it("detects German and English speech hooks", () => { + expect(speechHookScore("")).toBe(0); + expect(speechHookScore("the weather was fine")).toBe(0); + expect(speechHookScore("Wait, you won't believe this!", "en")).toBeGreaterThan(0.6); + expect(speechHookScore("Warum macht das niemand?", "de")).toBeGreaterThan(0.5); + expect(speechHookScore("Schau mal, 3 Tricks!", "de-DE")).toBeGreaterThan(0.6); + expect(speechHookScore("Here are 5 tips")).toBeGreaterThan(speechHookScore("here are tips")); + expect(speechHookScore("Wie geht das? Pass auf! 10 Sekunden, du!")).toBe(1); + }); +}); + +describe("selectTeaserSegments", () => { + const profile = testProfile(); + const select = (overrides: Partial[0]> = {}) => + selectTeaserSegments({ + segments, + profile, + goal: "teaser", + targetMs: 15_000, + weights: defaultTeaserWeights("teaser"), + endCardMs: 2000, + ...overrides, + }); + + it("opens with the hook inside the hook window", () => { + const selection = select(); + expect(selection.hookSegmentId).toBe("s2"); + expect(selection.clips[0]).toMatchObject({ segmentId: "s2", inMs: 4000 }); + expect(selection.notes[0]).toContain("hook window"); + }); + + it("never uses paid-only segments and ends before the payoff", () => { + const selection = select(); + const ids = selection.clips.map((c) => c.segmentId); + expect(ids).not.toContain("s5"); + expect(ids).not.toContain("s6"); + expect(ids).not.toContain("s7"); + const last = selection.clips.at(-1)!; + expect(last.assetId).toBe("asset-a"); + expect(last.outMs).toBeLessThanOrEqual(20_000); + expect(selection.notes.some((n) => n.startsWith("open loop"))).toBe(true); + }); + + it("builds tension and fits the duration bounds minus the end card", () => { + const selection = select(); + const total = selection.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0); + expect(total).toBeLessThanOrEqual(15_000 - 2000); + expect(total).toBeGreaterThanOrEqual(profile.duration.hardMinMs - 2000); + for (const clip of selection.clips) expect(clip.outMs).toBeGreaterThan(clip.inMs); + const tight = select({ targetMs: 5000 }); + expect(tight.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0)).toBeLessThanOrEqual(3000); + }); + + it("returns nothing when only paid-only material exists", () => { + const paid = segments.map((s) => ({ ...s, paidOnly: true })); + expect(select({ segments: paid })).toEqual({ + clips: [], + hookSegmentId: null, + notes: ["no usable segments"], + }); + }); + + it("is deterministic", () => { + expect(select({ segments: [...segments].reverse() })).toEqual(select()); + }); +}); + +describe("fitTeaserWeights", () => { + const prior = defaultTeaserWeights("teaser"); + const samples = (count: number) => + Array.from({ length: count }, (_, i) => { + const hook = (i % 10) / 9; + const motion = ((i * 7) % 10) / 9; + return { + features: { speechHook: hook, motion }, + views: 1000, + conversions: Math.round(1000 * (0.01 + 0.2 * hook)), + }; + }); + + it("stays off below the minimum sample count", () => { + expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES - 1), prior)).toBeNull(); + const withoutViews = [ + ...samples(TEASER_LEARNING_MIN_SAMPLES - 1), + { features: {}, views: 0, conversions: 0 }, + ]; + expect(fitTeaserWeights(withoutViews, prior)).toBeNull(); + }); + + it("shifts weight toward features that drive conversion, deterministically", () => { + const learned = fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)!; + expect(learned).not.toBeNull(); + expect(learned.speechHook).toBeGreaterThan(prior.speechHook); + expect(learned.motion).toBeLessThan(prior.motion); + expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)).toEqual(learned); + const more = fitTeaserWeights(samples(200), prior)!; + expect(more.speechHook).toBeGreaterThan(learned.speechHook); + }); +}); + +describe("autoPlanEdl", () => { + const assets = new Map([ + ["asset-a", { durationMs: 30_000, width: 1920, height: 1080, hasSpeech: true }], + ["asset-b", { durationMs: 8000, width: 1080, height: 1920, hasSpeech: false }], + ]); + const context = testContext({ + assets: new Map([ + ["asset-a", { kind: "video", durationMs: 30_000, width: 1920, height: 1080 }], + ["asset-b", { kind: "video", durationMs: 8000, width: 1080, height: 1920 }], + ]), + paidOnly: [{ assetId: "asset-a", startMs: 16_000, endMs: 20_000 }], + templateIds: new Set(["intro-minimal", "end-card-a"]), + goal: "teaser", + }); + + it("plans a valid teaser with hook first, captions, watermark, CTA and redactions", () => { + const kit = testKit({ + intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 }, + endCard: { templateId: "end-card-a", params: { style: "a" }, durationMs: 2000 }, + lutArtifactId: "lut-1", + }); + const edl = autoPlanEdl({ + goal: "teaser", + profile: testProfile(), + brandKit: kit, + segments, + weights: defaultTeaserWeights("teaser"), + assets, + people: [ + { id: "person-1", anonymizeByDefault: "synthetic_identity" }, + { id: "person-2", anonymizeByDefault: "none" }, + ], + marks: [ + { id: "mark-1", treatment: "inpaint", assetId: "asset-a", personId: null }, + { id: "mark-2", treatment: "blur", assetId: "asset-z", personId: null }, + { id: "mark-3", treatment: "none", assetId: "asset-a", personId: null }, + ], + ctaText: "Full video: link in bio", + }); + const result = validateEdl(edl, { ...context, brandKit: kit }); + expect(result.ok ? [] : result.errors).toEqual([]); + // Short vertical profile: no intro, the hook comes first. + expect(edl.intro).toBeUndefined(); + expect(edl.tracks.video[0]!.segmentId).toBe("s2"); + expect(edl.tracks.video[0]!.reframe).toBe("auto"); + expect(edl.tracks.video[0]!.color?.useBrandLut).toBe(true); + expect(edl.outro).toMatchObject({ + templateId: "end-card-a", + params: { style: "a", cta: "Full video: link in bio" }, + }); + expect(edl.captions).toMatchObject({ mode: "both", presetId: "clean", maxCharsPerLine: 24 }); + expect(edl.overlays.map((o) => o.kind)).toEqual(["watermark"]); + expect(edl.redactions.faces.map((f) => [f.target, f.mode])).toEqual([ + [{ personId: "person-1" }, "synthetic_identity"], + [{ unassignedFaces: true }, "blur"], + ]); + expect(edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "inpaint" }]); + expect(edl.metadata).toMatchObject({ + aiAltered: true, + aiAlterations: ["synthetic_identity", "inpaint"], + }); + }); + + it("uses a CTA text card without an end-card template and keeps the intro on long-form profiles", () => { + const profile = testPaidProfile(); + const kit = testKit({ + intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 }, + watermark: undefined, + }); + const teaser = autoPlanEdl({ + goal: "teaser", + profile, + brandKit: kit, + segments, + weights: defaultTeaserWeights("teaser"), + assets, + people: [], + marks: [], + ctaText: "Watch the full video", + }); + expect(teaser.intro?.templateId).toBe("intro-minimal"); + expect(teaser.overlays).toEqual([ + expect.objectContaining({ kind: "cta_end_card", text: "Watch the full video" }), + ]); + expect(teaser.redactions.faces).toEqual([]); + expect(teaser.metadata.aiAltered).toBe(false); + const result = validateEdl(teaser, { ...context, profile, brandKit: kit }); + expect(result.ok ? [] : result.errors).toEqual([]); + }); + + it("plans a full release in capture order including paid-only material on paid profiles", () => { + const profile = testPaidProfile(); + const edl = autoPlanEdl({ + goal: "full_release", + profile, + brandKit: null, + segments, + weights: defaultTeaserWeights("full_release"), + assets, + people: [], + marks: [], + ctaText: null, + }); + expect(edl.tracks.video.map((c) => c.segmentId)).toEqual([ + "s1", + "s2", + "s3", + "s4", + "s5", + "s6", + "s7", + "s8", + ]); + expect(edl.tracks.video.at(-1)!.reframe).toBe("auto"); + expect(edl.tracks.video[0]!.reframe).toBe("none"); + const result = validateEdl(edl, { ...context, profile, brandKit: null, goal: "full_release" }); + expect(result.ok ? [] : result.errors).toEqual([]); + expect(() => + autoPlanEdl({ + goal: "teaser", + profile, + brandKit: null, + segments: [], + weights: defaultTeaserWeights("teaser"), + assets, + people: [], + marks: [], + ctaText: null, + }), + ).toThrow(/no usable segments/); + }); +}); + +describe("rankThumbnailFrames", () => { + it("returns the best frames spread over the video", () => { + const candidates = [ + { tMs: 0, sharpness: 0.5, face: 0, thirds: 0.5, contrast: 0.5 }, + { tMs: 1000, sharpness: 1, face: 1, thirds: 1, contrast: 1 }, + { tMs: 1200, sharpness: 1, face: 1, thirds: 0.9, contrast: 1 }, + { tMs: 5000, sharpness: 0.8, face: 0.5, thirds: 0.5, contrast: 0.5 }, + { tMs: 9000, sharpness: 0.7, face: 0.5, thirds: 0.5, contrast: 0.5 }, + ]; + expect(rankThumbnailFrames(candidates, 3)).toEqual([1000, 5000, 9000]); + expect(rankThumbnailFrames(candidates.slice(1, 3), 3)).toEqual([1000, 1200]); + expect(rankThumbnailFrames(candidates, 0)).toEqual([]); + }); +}); diff --git a/packages/media/src/teaser.ts b/packages/media/src/teaser.ts new file mode 100644 index 0000000..e8c7818 --- /dev/null +++ b/packages/media/src/teaser.ts @@ -0,0 +1,659 @@ +import type { + AiAlteration, + AnonymizeMode, + BrandKit, + Edl, + MediaProjectGoal, + PlatformProfile, + TeaserFeature, + TeaserWeights, +} from "@rakazo/contracts"; +import { + AnonymizeModeSchema, + EdlSchema, + TEASER_FEATURES, + TEASER_LEARNING_MIN_SAMPLES, +} from "@rakazo/contracts"; +import { introFits } from "./marketing.js"; + +export type SegmentFeatures = Partial>; + +const GOAL_WEIGHTS: Record> = { + teaser: { + motion: 0.14, + sharpness: 0.06, + faces: 0.08, + gaze: 0.08, + audioPeak: 0.14, + speechHook: 0.24, + novelty: 0.12, + beatProximity: 0.06, + aesthetics: 0.08, + }, + trailer: { + motion: 0.16, + sharpness: 0.08, + faces: 0.08, + gaze: 0.06, + audioPeak: 0.14, + speechHook: 0.12, + novelty: 0.12, + beatProximity: 0.12, + aesthetics: 0.12, + }, + highlight: { + motion: 0.2, + sharpness: 0.08, + faces: 0.12, + gaze: 0.06, + audioPeak: 0.18, + speechHook: 0.08, + novelty: 0.1, + beatProximity: 0.08, + aesthetics: 0.1, + }, + story: { + motion: 0.08, + sharpness: 0.08, + faces: 0.16, + gaze: 0.12, + audioPeak: 0.06, + speechHook: 0.22, + novelty: 0.12, + beatProximity: 0.04, + aesthetics: 0.12, + }, + full_release: { + motion: 0.1, + sharpness: 0.14, + faces: 0.1, + gaze: 0.08, + audioPeak: 0.08, + speechHook: 0.1, + novelty: 0.12, + beatProximity: 0.06, + aesthetics: 0.22, + }, + custom: { + motion: 0.12, + sharpness: 0.1, + faces: 0.1, + gaze: 0.1, + audioPeak: 0.12, + speechHook: 0.14, + novelty: 0.12, + beatProximity: 0.08, + aesthetics: 0.12, + }, +}; + +/** Default per-goal weights over every teaser feature; each set sums to 1. */ +export function defaultTeaserWeights(goal: MediaProjectGoal): TeaserWeights { + return { ...GOAL_WEIGHTS[goal] }; +} + +export interface ScoredSegmentInput { + id: string; + assetId: string; + startMs: number; + endMs: number; + capturedAt: Date | null; + features: SegmentFeatures; + paidOnly: boolean; + transcript: string | null; +} + +const clamp01 = (value: number) => (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0); + +/** Weighted mean of 0..1 features; missing features count as 0. */ +export function scoreSegment(features: SegmentFeatures, weights: TeaserWeights): number { + let total = 0; + let norm = 0; + for (const feature of TEASER_FEATURES) { + const weight = weights[feature] ?? 0; + norm += Math.abs(weight); + total += weight * clamp01(features[feature] ?? 0); + } + return norm === 0 ? 0 : total / norm; +} + +const HOOK_LEXICON = { + en: { + questionWords: + /^(what|why|how|who|when|where|which|did|do|does|can|could|would|should|is|are|have|ever)\b/, + announcements: + /\b(wait|watch|look|listen|check this|you won'?t believe|here'?s|secret|how to|never|finally|the moment|let me show|guess|imagine|stop|nobody|everyone|biggest|best|worst|first time|new|today)\b/, + address: /\b(you|your)\b/, + numbers: /\b(one|two|three|five|ten|hundred|thousand|million|first|second|third|top)\b/, + }, + de: { + questionWords: + /^(was|warum|wieso|weshalb|wie|wer|wann|wo|welche[rsmn]?|hast|habt|kannst|könnt|willst|würdest|weißt|ist|sind)\b/, + announcements: + /\b(warte|wartet|schau|schaut|guck|hör zu|pass auf|achtung|du wirst nicht glauben|geheimnis|so geht|niemals|endlich|gleich|hier ist|zeig ich dir|stell dir vor|rate mal|niemand|alle|größte|beste|schlimmste|zum ersten mal|neu|heute)\b/, + address: /\b(du|dich|dir|dein|deine|ihr|euch|euer)\b/, + numbers: /\b(eins|zwei|drei|fünf|zehn|hundert|tausend|million|erste[rsn]?|zweite[rsn]?|top)\b/, + }, +} as const; + +/** 0..1 speech-hook strength from a transcript excerpt (questions, exclamations, numbers, announcements). */ +export function speechHookScore(text: string, language?: string): number { + const normalized = text.trim().toLocaleLowerCase(language); + if (!normalized) return 0; + const code = language?.slice(0, 2).toLowerCase(); + const lexicons = + code === "de" + ? [HOOK_LEXICON.de] + : code === "en" + ? [HOOK_LEXICON.en] + : Object.values(HOOK_LEXICON); + let score = 0; + if (normalized.includes("?")) score += 0.35; + if (normalized.includes("!")) score += 0.25; + const hasNumber = + /\d/.test(normalized) || lexicons.some((lexicon) => lexicon.numbers.test(normalized)); + if (hasNumber) score += 0.2; + if (lexicons.some((lexicon) => lexicon.announcements.test(normalized))) score += 0.3; + if (lexicons.some((lexicon) => lexicon.questionWords.test(normalized))) score += 0.15; + if (lexicons.some((lexicon) => lexicon.address.test(normalized))) score += 0.1; + return Math.min(1, Math.round(score * 1000) / 1000); +} + +function segmentFeatures(segment: ScoredSegmentInput): SegmentFeatures { + if (segment.features.speechHook !== undefined || !segment.transcript) return segment.features; + return { ...segment.features, speechHook: speechHookScore(segment.transcript) }; +} + +/** Strength of a segment as an opener: speech hooks, audio peaks, motion, eye contact. */ +function hookStrength(features: SegmentFeatures): number { + return ( + 0.45 * clamp01(features.speechHook ?? 0) + + 0.25 * clamp01(features.audioPeak ?? 0) + + 0.15 * clamp01(features.motion ?? 0) + + 0.15 * clamp01(features.gaze ?? 0) + ); +} + +export interface TeaserSelection { + clips: Array<{ segmentId: string; assetId: string; inMs: number; outMs: number }>; + hookSegmentId: string | null; + /** Why each clip was chosen, for plan rationale. */ + notes: string[]; +} + +const MIN_CLIP_MS = 1000; + +/** + * Pick clips for a goal and profile: a hook inside hookWindowMs, rising + * tension, cut before the payoff (open loop), never paid-only segments, + * total within the profile's duration bounds. + * + * The primary asset is the one holding the best segment overall; its payoff + * is its best segment (paid-only included, since that is usually the payoff). + * Nothing from the primary asset at or after the payoff is used. The hook + * opens the teaser, the rest follows in ascending score, and the lead-in to + * the payoff closes it, leaving `endCardMs` for the end card. + */ +export function selectTeaserSegments(input: { + segments: ScoredSegmentInput[]; + profile: PlatformProfile; + goal: MediaProjectGoal; + targetMs: number; + weights: TeaserWeights; + endCardMs: number; +}): TeaserSelection { + const { profile } = input; + const notes: string[] = []; + const scored = input.segments + .filter((s) => s.endMs > s.startMs) + .map((segment) => { + const features = segmentFeatures(segment); + return { segment, features, score: scoreSegment(features, input.weights) }; + }) + .sort( + (a, b) => + b.score - a.score || + a.segment.assetId.localeCompare(b.segment.assetId) || + a.segment.startMs - b.segment.startMs || + a.segment.id.localeCompare(b.segment.id), + ); + const maxMs = Math.max( + 0, + Math.min(input.targetMs, profile.duration.hardMaxMs) - Math.max(0, input.endCardMs), + ); + const minMs = Math.max(0, profile.duration.hardMinMs - Math.max(0, input.endCardMs)); + const usable = scored.filter((s) => !s.segment.paidOnly); + if (usable.length === 0 || maxMs < MIN_CLIP_MS) { + return { clips: [], hookSegmentId: null, notes: ["no usable segments"] }; + } + + const primaryAsset = scored[0]!.segment.assetId; + const payoff = scored.find((s) => s.segment.assetId === primaryAsset)!; + const pool = usable.filter( + (s) => + s.segment.assetId !== primaryAsset || + (s.segment.id !== payoff.segment.id && s.segment.endMs <= payoff.segment.startMs), + ); + const openLoop = pool.length < usable.length || payoff.segment.paidOnly; + if (pool.length === 0) { + return { clips: [], hookSegmentId: null, notes: ["only the payoff is usable"] }; + } + + const hook = [...pool].sort( + (a, b) => + hookStrength(b.features) - hookStrength(a.features) || + b.score - a.score || + a.segment.id.localeCompare(b.segment.id), + )[0]!; + const hookWindow = profile.duration.hookWindowMs; + const clipCap = Math.max(MIN_CLIP_MS, Math.min(5000, Math.round(maxMs / 4))); + const lengthOf = (s: ScoredSegmentInput, cap: number) => Math.min(s.endMs - s.startMs, cap); + + // Lead-in: the last primary-asset segment before the payoff closes the teaser. + const leadIn = openLoop + ? pool + .filter((s) => s.segment.assetId === primaryAsset && s !== hook) + .sort((a, b) => b.segment.endMs - a.segment.endMs)[0] + : undefined; + + const chosen: typeof pool = [hook]; + let total = lengthOf(hook.segment, Math.max(clipCap, Math.min(hookWindow, maxMs))); + const reserve = leadIn ? lengthOf(leadIn.segment, clipCap) : 0; + for (const candidate of pool) { + if (candidate === hook || candidate === leadIn) continue; + const length = lengthOf(candidate.segment, clipCap); + if (length < Math.min(MIN_CLIP_MS, candidate.segment.endMs - candidate.segment.startMs)) + continue; + if (total + length + reserve > maxMs) continue; + chosen.push(candidate); + total += length; + } + if (leadIn && total + reserve <= maxMs) { + chosen.push(leadIn); + total += reserve; + } + + const middle = chosen + .slice(1) + .filter((s) => s !== leadIn) + .sort((a, b) => a.score - b.score || a.segment.id.localeCompare(b.segment.id)); + const ordered = [hook, ...middle, ...(chosen.includes(leadIn!) ? [leadIn!] : [])]; + + // Lengths: start from the cap, then grow toward the minimum duration if needed. + const lengths = ordered.map((s, i) => + lengthOf(s.segment, i === 0 ? Math.max(clipCap, Math.min(hookWindow, maxMs)) : clipCap), + ); + let sum = lengths.reduce((a, b) => a + b, 0); + for (let i = 0; i < ordered.length && sum < minMs; i++) { + const full = ordered[i]!.segment.endMs - ordered[i]!.segment.startMs; + const grow = Math.min(full - lengths[i]!, minMs - sum, maxMs - sum); + if (grow > 0) { + lengths[i]! += grow; + sum += grow; + } + } + while (sum > maxMs && lengths.length > 0) { + const over = sum - maxMs; + const last = lengths.length - 1; + if (lengths[last]! - over >= MIN_CLIP_MS) { + lengths[last]! -= over; + sum -= over; + } else { + sum -= lengths.pop()!; + ordered.pop(); + } + } + + const clips = ordered.map((s, i) => { + const length = lengths[i]!; + // The lead-in ends where the payoff would begin; other clips open at their start. + const isLeadIn = s === leadIn; + const outMs = isLeadIn ? s.segment.endMs : s.segment.startMs + length; + return { + segmentId: s.segment.id, + assetId: s.segment.assetId, + inMs: outMs - length, + outMs, + }; + }); + + notes.push( + `hook: segment ${hook.segment.id} opens inside the ${hookWindow} ms hook window`, + ...middle.map((s) => `build: segment ${s.segment.id} (score ${s.score.toFixed(2)})`), + ); + if (leadIn && ordered.includes(leadIn)) { + notes.push(`open loop: segment ${leadIn.segment.id} cuts before payoff ${payoff.segment.id}`); + } else if (openLoop) { + notes.push(`open loop: payoff ${payoff.segment.id} is left out`); + } + if (input.segments.some((s) => s.paidOnly)) notes.push("paid-only segments excluded"); + if (sum < minMs) notes.push(`only ${sum} ms of usable material; profile minimum is ${minMs} ms`); + return { clips, hookSegmentId: hook.segment.id, notes }; +} + +const LEARNING_RATE = 0.5; +const LEARNING_ITERATIONS = 400; +const L2 = 0.05; +const SHRINKAGE = 50; + +/** + * Regularized logistic regression of conversion on features; null below + * TEASER_LEARNING_MIN_SAMPLES samples with views. + * + * Each sample contributes `conversions` successes out of `views` trials, + * weighted so every publication counts equally. Full-batch gradient descent + * from zero with a fixed iteration count keeps it deterministic. Positive + * coefficients of features that vary in the data redistribute those features' + * prior mass; features without variation keep their prior. The result is + * shrunk toward the prior by n / (n + 50) so a small sample cannot overturn it. + */ +export function fitTeaserWeights( + samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>, + prior: TeaserWeights, +): TeaserWeights | null { + const usable = samples.filter((s) => s.views > 0 && s.conversions >= 0); + if (usable.length < TEASER_LEARNING_MIN_SAMPLES) return null; + const k = TEASER_FEATURES.length; + const xs = usable.map((s) => TEASER_FEATURES.map((f) => clamp01(s.features[f] ?? 0))); + const ys = usable.map((s) => Math.min(1, s.conversions / s.views)); + const beta = new Array(k).fill(0); + let intercept = 0; + for (let iteration = 0; iteration < LEARNING_ITERATIONS; iteration++) { + const gradient = new Array(k).fill(0); + let gradientIntercept = 0; + for (let i = 0; i < xs.length; i++) { + const x = xs[i]!; + let z = intercept; + for (let j = 0; j < k; j++) z += beta[j]! * x[j]!; + const error = 1 / (1 + Math.exp(-z)) - ys[i]!; + gradientIntercept += error; + for (let j = 0; j < k; j++) gradient[j]! += error * x[j]!; + } + intercept -= (LEARNING_RATE * gradientIntercept) / xs.length; + for (let j = 0; j < k; j++) { + beta[j]! -= LEARNING_RATE * (gradient[j]! / xs.length + L2 * beta[j]!); + } + } + // Only features that vary in the data are re-weighted; they share their prior mass. + const varies = TEASER_FEATURES.map((_, j) => xs.some((x) => x[j] !== xs[0]![j])); + const mass = TEASER_FEATURES.reduce( + (sum, f, j) => sum + (varies[j] ? Math.max(0, prior[f] ?? 0) : 0), + 0, + ); + const positive = beta.map((b, j) => (varies[j] ? Math.max(0, b) : 0)); + const positiveTotal = positive.reduce((a, b) => a + b, 0); + const alpha = usable.length / (usable.length + SHRINKAGE); + return Object.fromEntries( + TEASER_FEATURES.map((feature, j) => { + const base = prior[feature] ?? 0; + const learned = varies[j] && positiveTotal > 0 ? (positive[j]! / positiveTotal) * mass : base; + const value = (1 - alpha) * base + alpha * learned; + return [feature, Math.round(value * 1e6) / 1e6]; + }), + ) as TeaserWeights; +} + +const CTA_CARD_MS = 2500; + +function pickTarget(profile: PlatformProfile, goal: MediaProjectGoal, availableMs: number): number { + const { hardMinMs, hardMaxMs, recommended } = profile.duration; + const options = [...recommended] + .filter((r) => r >= hardMinMs && r <= hardMaxMs) + .sort((a, b) => a - b); + let target: number; + if (goal === "full_release" || goal === "custom") { + target = Math.min(hardMaxMs, availableMs); + } else if (options.length > 0) { + target = options[Math.floor((options.length - 1) / 2)]!; + } else { + target = Math.min(hardMaxMs, 30_000); + } + return Math.max(hardMinMs, Math.min(hardMaxMs, target)); +} + +/** + * Deterministic first-cut EDL for a project and profile; the agent refines it. + * + * Teaser-style goals use selectTeaserSegments; full releases and custom + * projects take segments in order (paid-only ones only for non-social + * profiles). Reframing is "auto" whenever the source aspect differs from the + * output, captions follow the profile when the material has speech, the + * brand intro only plays where introFits allows it, and a teaser ends on the + * brand end card (or a CTA text card). Person and mark defaults become + * redactions, and the AI-alteration metadata follows from them. + */ +export function autoPlanEdl(input: { + goal: MediaProjectGoal; + profile: PlatformProfile; + brandKit: BrandKit | null; + segments: ScoredSegmentInput[]; + weights: TeaserWeights; + assets: Map< + string, + { durationMs: number | null; width: number | null; height: number | null; hasSpeech: boolean } + >; + /** Person anonymization defaults to carry into redactions. */ + people: Array<{ id: string; anonymizeByDefault: string }>; + marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; + ctaText: string | null; +}): Edl { + const { goal, profile, brandKit: kit } = input; + const W = profile.video.width; + const H = profile.video.height; + const segments = input.segments.filter((s) => { + const asset = input.assets.get(s.assetId); + return ( + asset && s.endMs > s.startMs && (asset.durationMs === null || s.endMs <= asset.durationMs) + ); + }); + const available = segments.reduce((sum, s) => sum + (s.endMs - s.startMs), 0); + const targetMs = pickTarget(profile, goal, available); + + const intro = kit?.intro && introFits(profile, kit.intro.durationMs) ? kit.intro : undefined; + let outro = goal === "teaser" ? undefined : kit?.outro; + const ctaText = input.ctaText?.trim() || null; + let ctaCard = false; + if (goal === "teaser") { + if (kit?.endCard) { + outro = { + ...kit.endCard, + params: { ...kit.endCard.params, ...(ctaText ? { cta: ctaText.slice(0, 200) } : {}) }, + }; + } else if (ctaText) { + ctaCard = true; + } + } + const bookendsMs = (intro?.durationMs ?? 0) + (outro?.durationMs ?? 0); + const budget = Math.max(MIN_CLIP_MS, targetMs - bookendsMs); + + let picks: Array<{ segmentId: string; assetId: string; inMs: number; outMs: number }>; + if (goal === "full_release" || goal === "custom") { + const allowPaid = profile.kind !== "free_social"; + const ordered = segments + .filter((s) => allowPaid || !s.paidOnly) + .sort( + (a, b) => + (a.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) - + (b.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) || + a.assetId.localeCompare(b.assetId) || + a.startMs - b.startMs, + ); + picks = []; + let used = 0; + for (const s of ordered) { + const length = Math.min(s.endMs - s.startMs, budget - used); + if (length < MIN_CLIP_MS) break; + picks.push({ + segmentId: s.id, + assetId: s.assetId, + inMs: s.startMs, + outMs: s.startMs + length, + }); + used += length; + } + } else { + picks = selectTeaserSegments({ + segments, + profile, + goal, + targetMs: budget, + weights: input.weights, + endCardMs: 0, + }).clips; + } + if (picks.length === 0) throw new Error("no usable segments for this plan"); + + let atMs = 0; + const lut = Boolean(kit?.lutArtifactId); + const video = picks.map((pick, index) => { + const asset = input.assets.get(pick.assetId)!; + const sameAspect = + !asset.width || !asset.height || Math.abs(asset.width / asset.height / (W / H) - 1) <= 0.01; + const clip = { + id: `c${index + 1}`, + assetId: pick.assetId, + inMs: pick.inMs, + outMs: pick.outMs, + atMs, + speed: 1, + reframe: sameAspect ? ("none" as const) : ("auto" as const), + ...(lut ? { color: { useBrandLut: true } } : {}), + segmentId: pick.segmentId, + }; + atMs += pick.outMs - pick.inMs; + return clip; + }); + const mainMs = atMs; + const introLength = intro?.durationMs ?? 0; + const totalMs = introLength + mainMs + (outro?.durationMs ?? 0); + + const usedAssets = new Set(picks.map((p) => p.assetId)); + const hasSpeech = [...usedAssets].some((id) => input.assets.get(id)?.hasSpeech); + const preset = kit?.captionPresets[0]; + const captions = + (profile.captions.recommended || profile.captions.required) && hasSpeech + ? { + mode: profile.export.subtitles ? ("both" as const) : ("burned" as const), + ...(preset ? { presetId: preset.id } : {}), + maxCharsPerLine: W < H ? 24 : 32, + wordHighlight: preset?.template === "karaoke", + position: "lower" as const, + } + : undefined; + + const zones = profile.safeZones; + const overlays: Array> = []; + if (kit?.watermark) { + overlays.push({ id: "watermark", kind: "watermark", atMs: 0, durationMs: totalMs }); + } + if (ctaCard && ctaText) { + const cardMs = Math.min(CTA_CARD_MS, mainMs); + const left = zones.leftPct / 100 + 0.02; + const right = zones.rightPct / 100 + 0.02; + overlays.push({ + id: "cta", + kind: "cta_end_card", + atMs: introLength + mainMs - cardMs, + durationMs: cardMs, + text: ctaText.slice(0, 200), + box: { x: left, y: 0.42, w: Math.max(0.1, 1 - left - right), h: 0.12 }, + }); + } + + const protectedPeople = input.people.flatMap((person) => { + const mode = AnonymizeModeSchema.safeParse(person.anonymizeByDefault); + return mode.success && mode.data !== "none" ? [{ id: person.id, mode: mode.data }] : []; + }); + const faces: Array<{ + id: string; + target: { personId: string } | { unassignedFaces: true }; + mode: Exclude; + }> = protectedPeople.map((person, index) => ({ + id: `face-${index + 1}`, + target: { personId: person.id }, + mode: person.mode, + })); + if (faces.length > 0) { + // Unknown faces next to a protected person are blurred too; the gate blocks them otherwise. + faces.push({ id: "face-unassigned", target: { unassignedFaces: true }, mode: "blur" }); + } + const peopleIds = new Set(input.people.map((p) => p.id)); + const marks = input.marks.flatMap((mark) => { + if (mark.treatment === "none") return []; + const relevant = + (mark.assetId !== null && usedAssets.has(mark.assetId)) || + (mark.personId !== null && peopleIds.has(mark.personId)) || + (mark.assetId === null && mark.personId === null); + if (!relevant) return []; + const treatment = mark.treatment as "inpaint" | "blur" | "cover" | "reframe_out"; + return [{ markId: mark.id, treatment }]; + }); + + const aiAlterations: AiAlteration[] = []; + if (faces.some((f) => f.mode === "synthetic_identity")) aiAlterations.push("synthetic_identity"); + if (marks.some((m) => m.treatment === "inpaint")) aiAlterations.push("inpaint"); + + return EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: profile.id, + width: W, + height: H, + fps: profile.video.fps, + durationTargetMs: targetMs, + durationHardMinMs: profile.duration.hardMinMs, + durationHardMaxMs: profile.duration.hardMaxMs, + }, + tracks: { video, audio: [] }, + transitions: [], + ...(captions ? { captions } : {}), + overlays, + ...(intro ? { intro } : {}), + ...(outro ? { outro } : {}), + redactions: { faces, marks, regions: [] }, + metadata: { tags: [], aiAltered: aiAlterations.length > 0, aiAlterations }, + }); +} + +const THUMBNAIL_WEIGHTS = { sharpness: 0.4, face: 0.25, thirds: 0.2, contrast: 0.15 }; + +/** + * Best thumbnail frame times, spread out: candidates are ranked by sharpness, + * face presence, rule of thirds and contrast, and each pick must be at least + * a tenth of the candidate span (min 1 s) away from earlier picks while + * enough candidates remain. + */ +export function rankThumbnailFrames( + candidates: Array<{ + tMs: number; + sharpness: number; + face: number; + thirds: number; + contrast: number; + }>, + count: number, +): number[] { + if (count <= 0 || candidates.length === 0) return []; + const ranked = candidates + .map((c) => ({ + tMs: c.tMs, + score: + THUMBNAIL_WEIGHTS.sharpness * clamp01(c.sharpness) + + THUMBNAIL_WEIGHTS.face * clamp01(c.face) + + THUMBNAIL_WEIGHTS.thirds * clamp01(c.thirds) + + THUMBNAIL_WEIGHTS.contrast * clamp01(c.contrast), + })) + .sort((a, b) => b.score - a.score || a.tMs - b.tMs); + const times = candidates.map((c) => c.tMs); + const spacing = Math.max(1000, (Math.max(...times) - Math.min(...times)) / 10); + const picked: number[] = []; + for (const candidate of ranked) { + if (picked.length >= count) break; + if (picked.every((t) => Math.abs(t - candidate.tMs) >= spacing)) picked.push(candidate.tMs); + } + for (const candidate of ranked) { + if (picked.length >= count) break; + if (!picked.includes(candidate.tMs)) picked.push(candidate.tMs); + } + return picked; +} diff --git a/packages/media/src/test-fixtures.ts b/packages/media/src/test-fixtures.ts new file mode 100644 index 0000000..105843f --- /dev/null +++ b/packages/media/src/test-fixtures.ts @@ -0,0 +1,135 @@ +import type { BrandKit, Edl, PlatformProfile } from "@rakazo/contracts"; +import { BrandKitSchema, EdlSchema, PlatformProfileSchema } from "@rakazo/contracts"; +import type { EdlContext } from "./edl.js"; + +/* Synthetic fixtures shared by the unit tests. No real people, places, or platforms. */ + +export function testProfile(overrides: Record = {}): PlatformProfile { + return PlatformProfileSchema.parse({ + id: "example-short-vertical", + name: "Example short vertical", + kind: "free_social", + reviewedAt: "2026-09-01", + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 500, + }, + duration: { hardMinMs: 1000, hardMaxMs: 60_000, recommended: [7000, 15_000, 30_000] }, + safeZones: { topPct: 10, bottomPct: 20, leftPct: 5, rightPct: 12 }, + captions: { recommended: true, maxLines: 2 }, + audio: {}, + text: { + titleMaxChars: 100, + captionMaxChars: 150, + hashtagLimit: 3, + linkPolicy: "bio_only", + bannedPatterns: [{ pattern: "free\\s+money", reason: "misleading claim" }], + }, + content: { thresholds: { explicit_nudity: 0.2, violence: 0.5 } }, + disclosure: { aiAlteredLabelRequired: true }, + consent: { requirePerformerConsent: false, requireAgeVerification: false }, + export: {}, + ...overrides, + }); +} + +export function testPaidProfile(): PlatformProfile { + return testProfile({ + id: "example-paid-video", + name: "Example paid video", + kind: "paid_video", + video: { + aspectRatios: ["16:9"], + width: 1920, + height: 1080, + fps: 30, + codec: "h264", + maxBitrateKbps: 12_000, + maxFileSizeMb: 4000, + }, + duration: { hardMinMs: 1000, hardMaxMs: 3_600_000, recommended: [] }, + safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 }, + text: { captionMaxChars: 2000, hashtagLimit: 10, linkPolicy: "allowed" }, + consent: { requirePerformerConsent: true, requireAgeVerification: true }, + disclosure: { aiAlteredLabelRequired: true, ageRestrictedFlag: true }, + }); +} + +export function testKit(overrides: Partial = {}): BrandKit { + return BrandKitSchema.parse({ + name: "Studio Example", + colors: { + primary: "#112233", + secondary: "#445566", + accent: "#FFAA00", + text: "#FFFFFF", + background: "#000000", + }, + fonts: [{ family: "Example Sans", artifactId: "font-1", role: "caption", license: "OFL" }], + watermark: { artifactId: "logo-1", position: "top_right", opacity: 0.6, scale: 0.15 }, + captionPresets: [ + { + id: "clean", + name: "Clean", + template: "clean", + fontFamily: "Example Sans", + sizePct: 5, + textColor: "#FFFFFF", + }, + ], + musicBeds: [{ artifactId: "music-1", title: "Bed", license: "own", commercialUse: true }], + ...overrides, + }); +} + +/** Two 16:9 clips of a 10 s asset, 4 s total, on a vertical profile. */ +export function testEdl(overrides: Record = {}): Edl { + return EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: "example-short-vertical", + width: 1080, + height: 1920, + fps: 30, + durationTargetMs: 4000, + durationHardMinMs: 1000, + durationHardMaxMs: 60_000, + }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 2000, reframe: "auto" }, + ], + }, + metadata: {}, + ...overrides, + }); +} + +export function testContext(overrides: Partial = {}): EdlContext { + return { + profile: testProfile(), + brandKit: testKit(), + assets: new Map([ + ["asset-a", { kind: "video", durationMs: 10_000, width: 1920, height: 1080 }], + ["asset-b", { kind: "video", durationMs: 8000, width: 1920, height: 1080 }], + ["photo-1", { kind: "photo", durationMs: null, width: 4000, height: 3000 }], + ]), + artifactIds: new Set(["music-1", "music-2", "vo-1"]), + music: new Map([ + ["music-1", { commercialUse: true }], + ["music-2", { commercialUse: false }], + ]), + markIds: new Set(["mark-1"]), + personIds: new Set(["person-1"]), + faceTrackIds: new Set(["track-1"]), + templateIds: new Set(["intro-minimal", "outro-simple", "lower-third-a"]), + paidOnly: [], + ...overrides, + }; +} diff --git a/packages/media/tsconfig.json b/packages/media/tsconfig.json new file mode 100644 index 0000000..c699328 --- /dev/null +++ b/packages/media/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "noEmit": true + }, + "include": ["src"] +} diff --git a/packages/testkit/src/pin-test-env.ts b/packages/testkit/src/pin-test-env.ts index 9d96330..25cad9d 100644 --- a/packages/testkit/src/pin-test-env.ts +++ b/packages/testkit/src/pin-test-env.ts @@ -10,6 +10,8 @@ if (!process.env.VERIFY_PROVIDERS) { process.env.WAKEUP_DRIVER = "memory"; delete process.env.COMPOSIO_API_KEY; delete process.env.INTEGRATIONS_CATALOG_URL; + delete process.env.MEDIA_WORKER_URL; + delete process.env.MEDIA_WORKER_TOKEN; if (!process.env.VERIFY_DATABASE) { delete process.env.DATABASE_URL; delete process.env.REALTIME_DATABASE_URL; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2acb6e..51dbf6b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,9 @@ importers: '@rakazo/logging': specifier: workspace:* version: link:../../packages/logging + '@rakazo/media': + specifier: workspace:* + version: link:../../packages/media '@rakazo/memory': specifier: workspace:* version: link:../../packages/memory @@ -176,6 +179,9 @@ importers: expo-localization: specifier: 57.0.2 version: 57.0.2(expo@57.0.26)(react@19.2.3) + expo-media-library: + specifier: 57.0.5 + version: 57.0.5(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3)) expo-modules-core: specifier: 57.0.20 version: 57.0.20(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3) @@ -203,6 +209,9 @@ importers: expo-updates: specifier: 57.0.24 version: 57.0.24(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3) + expo-video: + specifier: 57.0.5 + version: 57.0.5(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3) react: specifier: 19.2.3 version: 19.2.3 @@ -288,6 +297,9 @@ importers: '@rakazo/ui-web': specifier: workspace:* version: link:../../packages/ui-web + '@tanstack/react-virtual': + specifier: 3.14.13 + version: 3.14.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3) better-auth: specifier: 1.7.4 version: 1.7.4(@opentelemetry/api@1.9.1)(@prisma/client@7.10.0(prisma@7.10.0(@types/react-dom@19.2.7(@types/react@19.2.18))(@types/react@19.2.18)(magicast@0.5.4)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(typescript@7.0.2))(typescript@7.0.2))(mysql2@3.15.3)(pg@8.23.0)(prisma@7.10.0(@types/react-dom@19.2.7(@types/react@19.2.18))(@types/react@19.2.18)(magicast@0.5.4)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(typescript@7.0.2))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.5.0)(jsdom@30.0.1(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@26.5.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0))) @@ -579,6 +591,12 @@ importers: '@rakazo/logging': specifier: workspace:* version: link:../logging + '@rakazo/media': + specifier: workspace:* + version: link:../media + '@resvg/resvg-js': + specifier: 2.6.2 + version: 2.6.2 chat: specifier: 4.40.0 version: 4.40.0(zod@4.6.2) @@ -764,6 +782,9 @@ importers: '@rakazo/logging': specifier: workspace:* version: link:../logging + '@rakazo/media': + specifier: workspace:* + version: link:../media dotenv: specifier: 17.4.2 version: 17.4.2 @@ -800,6 +821,22 @@ importers: specifier: 4.1.11 version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.5.0)(jsdom@30.0.1(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@26.5.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)) + packages/media: + dependencies: + '@rakazo/contracts': + specifier: workspace:* + version: link:../contracts + zod: + specifier: 4.6.2 + version: 4.6.2 + devDependencies: + typescript: + specifier: 7.0.2 + version: 7.0.2 + vitest: + specifier: 4.1.11 + version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.5.0)(jsdom@30.0.1(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@26.5.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0)) + packages/memory: dependencies: '@rakazo/adapter-kit': @@ -3800,6 +3837,82 @@ packages: '@types/react': optional: true + '@resvg/resvg-js-android-arm-eabi@2.6.2': + resolution: {integrity: sha512-FrJibrAk6v29eabIPgcTUMPXiEz8ssrAk7TXxsiZzww9UTQ1Z5KAbFJs+Z0Ez+VZTYgnE5IQJqBcoSiMebtPHA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@resvg/resvg-js-android-arm64@2.6.2': + resolution: {integrity: sha512-VcOKezEhm2VqzXpcIJoITuvUS/fcjIw5NA/w3tjzWyzmvoCdd+QXIqy3FBGulWdClvp4g+IfUemigrkLThSjAQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@resvg/resvg-js-darwin-arm64@2.6.2': + resolution: {integrity: sha512-nmok2LnAd6nLUKI16aEB9ydMC6Lidiiq2m1nEBDR1LaaP7FGs4AJ90qDraxX+CWlVuRlvNjyYJTNv8qFjtL9+A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@resvg/resvg-js-darwin-x64@2.6.2': + resolution: {integrity: sha512-GInyZLjgWDfsVT6+SHxQVRwNzV0AuA1uqGsOAW+0th56J7Nh6bHHKXHBWzUrihxMetcFDmQMAX1tZ1fZDYSRsw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@resvg/resvg-js-linux-arm-gnueabihf@2.6.2': + resolution: {integrity: sha512-YIV3u/R9zJbpqTTNwTZM5/ocWetDKGsro0SWp70eGEM9eV2MerWyBRZnQIgzU3YBnSBQ1RcxRZvY/UxwESfZIw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@resvg/resvg-js-linux-arm64-gnu@2.6.2': + resolution: {integrity: sha512-zc2BlJSim7YR4FZDQ8OUoJg5holYzdiYMeobb9pJuGDidGL9KZUv7SbiD4E8oZogtYY42UZEap7dqkkYuA91pg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@resvg/resvg-js-linux-arm64-musl@2.6.2': + resolution: {integrity: sha512-3h3dLPWNgSsD4lQBJPb4f+kvdOSJHa5PjTYVsWHxLUzH4IFTJUAnmuWpw4KqyQ3NA5QCyhw4TWgxk3jRkQxEKg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@resvg/resvg-js-linux-x64-gnu@2.6.2': + resolution: {integrity: sha512-IVUe+ckIerA7xMZ50duAZzwf1U7khQe2E0QpUxu5MBJNao5RqC0zwV/Zm965vw6D3gGFUl7j4m+oJjubBVoftw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@resvg/resvg-js-linux-x64-musl@2.6.2': + resolution: {integrity: sha512-UOf83vqTzoYQO9SZ0fPl2ZIFtNIz/Rr/y+7X8XRX1ZnBYsQ/tTb+cj9TE+KHOdmlTFBxhYzVkP2lRByCzqi4jQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@resvg/resvg-js-win32-arm64-msvc@2.6.2': + resolution: {integrity: sha512-7C/RSgCa+7vqZ7qAbItfiaAWhyRSoD4l4BQAbVDqRRsRgY+S+hgS3in0Rxr7IorKUpGE69X48q6/nOAuTJQxeQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@resvg/resvg-js-win32-ia32-msvc@2.6.2': + resolution: {integrity: sha512-har4aPAlvjnLcil40AC77YDIk6loMawuJwFINEM7n0pZviwMkMvjb2W5ZirsNOZY4aDbo5tLx0wNMREp5Brk+w==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@resvg/resvg-js-win32-x64-msvc@2.6.2': + resolution: {integrity: sha512-ZXtYhtUr5SSaBrUDq7DiyjOFJqBVL/dOBN7N/qmi/pO0IgiWW/f/ue3nbvu9joWE5aAKDoIzy/CxsY0suwGosQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@resvg/resvg-js@2.6.2': + resolution: {integrity: sha512-xBaJish5OeGmniDj9cW5PRa/PtmuVU3ziqrbr5xJj901ZDN4TosrVaNZpEiLZAxdfnhAe7uQ7QFWfjPe9d9K2Q==} + engines: {node: '>= 10'} + '@rolldown/binding-android-arm-eabi@1.2.8': resolution: {integrity: sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4116,6 +4229,15 @@ packages: peerDependencies: vite: ^5.2.0 || ^6 || ^7 || ^8 + '@tanstack/react-virtual@3.14.13': + resolution: {integrity: sha512-JbDTAwtzZ99aOeCrAfW5EsE5KSq5RWh6Af2dtFwyLIIk48Ja7vm6n4axu/43T3vfjFnEGJalxJ1wyYjSQD6bSg==} + peerDependencies: + react: 19.2.3 + react-dom: 19.2.3 + + '@tanstack/virtual-core@3.17.11': + resolution: {integrity: sha512-+ILjvtHup6Y2hzQ6YzwMgX1Q+oQpxEGOXCEsCNaPoIP0VxMbizIBTmYTDtkerkIQS8/CbP1BRuyt8V/8BCsy1g==} + '@testcontainers/postgresql@12.1.0': resolution: {integrity: sha512-Pjf2VSVNirEPfz36nidyrVAnZvc2YhajOznY4VgyEsvfTd5qiMNOuPq96drREvxAUtXl5SFLX7vXj7sSq4aTcA==} @@ -6165,6 +6287,12 @@ packages: peerDependencies: expo: '*' + expo-media-library@57.0.5: + resolution: {integrity: sha512-amSLrLsS7Gl4nCqbc2PlVbHj5jK9kQLsSOQgJ2QasHCSHzYXw6Ui7aolxX5APg9P4TgYPE0yWn46ndifh52bRw==} + peerDependencies: + expo: '*' + react-native: '*' + expo-modules-autolinking@57.0.13: resolution: {integrity: sha512-Hj5NjZRlfccazhKab+wK19leRVUr1Y/PUERaRvegIbUSkq4GVxKq6lU9QHlKQyC16MdD/VS4iwsiLivB4o7lvQ==} hasBin: true @@ -6279,6 +6407,13 @@ packages: expo-dev-client: optional: true + expo-video@57.0.5: + resolution: {integrity: sha512-VK2o+d6rb+tVeL2DI6d0F58pRV8d8bctLpOrMJEHRYtMIK5iwvjjrRx2mciYN8/d04SgWC8TkI+wuZ8Sbp/yuA==} + peerDependencies: + expo: '*' + react: 19.2.3 + react-native: '*' + expo@57.0.26: resolution: {integrity: sha512-U1rsFwcuZNnU61Z/YLnnVC/ZqP0wpxh/rCKUMff3ot/rhLpxFY1zxEqu4vyaexyvBAB1mwpl9lA/ltcHBoQGyg==} hasBin: true @@ -13559,6 +13694,57 @@ snapshots: optionalDependencies: '@types/react': 19.2.18 + '@resvg/resvg-js-android-arm-eabi@2.6.2': + optional: true + + '@resvg/resvg-js-android-arm64@2.6.2': + optional: true + + '@resvg/resvg-js-darwin-arm64@2.6.2': + optional: true + + '@resvg/resvg-js-darwin-x64@2.6.2': + optional: true + + '@resvg/resvg-js-linux-arm-gnueabihf@2.6.2': + optional: true + + '@resvg/resvg-js-linux-arm64-gnu@2.6.2': + optional: true + + '@resvg/resvg-js-linux-arm64-musl@2.6.2': + optional: true + + '@resvg/resvg-js-linux-x64-gnu@2.6.2': + optional: true + + '@resvg/resvg-js-linux-x64-musl@2.6.2': + optional: true + + '@resvg/resvg-js-win32-arm64-msvc@2.6.2': + optional: true + + '@resvg/resvg-js-win32-ia32-msvc@2.6.2': + optional: true + + '@resvg/resvg-js-win32-x64-msvc@2.6.2': + optional: true + + '@resvg/resvg-js@2.6.2': + optionalDependencies: + '@resvg/resvg-js-android-arm-eabi': 2.6.2 + '@resvg/resvg-js-android-arm64': 2.6.2 + '@resvg/resvg-js-darwin-arm64': 2.6.2 + '@resvg/resvg-js-darwin-x64': 2.6.2 + '@resvg/resvg-js-linux-arm-gnueabihf': 2.6.2 + '@resvg/resvg-js-linux-arm64-gnu': 2.6.2 + '@resvg/resvg-js-linux-arm64-musl': 2.6.2 + '@resvg/resvg-js-linux-x64-gnu': 2.6.2 + '@resvg/resvg-js-linux-x64-musl': 2.6.2 + '@resvg/resvg-js-win32-arm64-msvc': 2.6.2 + '@resvg/resvg-js-win32-ia32-msvc': 2.6.2 + '@resvg/resvg-js-win32-x64-msvc': 2.6.2 + '@rolldown/binding-android-arm-eabi@1.2.8': optional: true @@ -13847,6 +14033,14 @@ snapshots: tailwindcss: 4.3.3 vite: 8.3.0(@types/node@26.5.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.13)(yaml@2.9.0) + '@tanstack/react-virtual@3.14.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': + dependencies: + '@tanstack/virtual-core': 3.17.11 + react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) + + '@tanstack/virtual-core@3.17.11': {} + '@testcontainers/postgresql@12.1.0': dependencies: testcontainers: 12.1.0 @@ -16233,6 +16427,11 @@ snapshots: expo: 57.0.26(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.16)(expo-router@57.0.24)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-webview@13.16.1(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-json-utils: 57.0.2 + expo-media-library@57.0.5(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3)): + dependencies: + expo: 57.0.26(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.16)(expo-router@57.0.24)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-webview@13.16.1(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3) + expo-modules-autolinking@57.0.13(typescript@6.0.3): dependencies: '@expo/require-utils': 57.0.5(typescript@6.0.3) @@ -16391,6 +16590,12 @@ snapshots: transitivePeerDependencies: - supports-color + expo-video@57.0.5(expo@57.0.26)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3): + dependencies: + expo: 57.0.26(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.16)(expo-router@57.0.24)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-webview@13.16.1(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + react: 19.2.3 + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3) + expo@57.0.26(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.16)(expo-router@57.0.24)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-webview@13.16.1(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3))(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7))(@types/react@19.2.18)(react@19.2.3))(react@19.2.3)(typescript@6.0.3): dependencies: '@babel/runtime': 7.29.7