diff --git a/packages/adapter-kit/src/index.ts b/packages/adapter-kit/src/index.ts index 43f7898..e39a95c 100644 --- a/packages/adapter-kit/src/index.ts +++ b/packages/adapter-kit/src/index.ts @@ -1,5 +1,6 @@ export * from "./background-jobs.js"; export * from "./cloud-agents.js"; export * from "./interfaces.js"; +export * from "./media.js"; export * from "./registry.js"; export * from "./types.js"; diff --git a/packages/adapter-kit/src/media.ts b/packages/adapter-kit/src/media.ts new file mode 100644 index 0000000..4f57c4c --- /dev/null +++ b/packages/adapter-kit/src/media.ts @@ -0,0 +1,229 @@ +import type { + MediaJobStatus, + MediaOperation, + MediaOutput, + MediaOutputKind, + MediaPresetId, +} from "@rakazo/contracts"; +import type { AdapterDescriptor } from "./types.js"; + +/** Bytes handed to or returned by a media backend. */ +export interface MediaFile { + bytes: Uint8Array; + mimeType: string; + name?: string; +} + +export interface GeneratedMedia extends MediaFile { + width?: number; + height?: number; + durationMs?: number; +} + +/** Reports progress as a fraction from 0 to 1. */ +export type MediaProgressListener = (fraction: number) => void; + +/** Free and total memory of the backend's GPU, when the backend can tell. */ +export interface GpuStatus { + vramFreeMb: number; + vramTotalMb: number; +} + +export interface ImageGenerationRequest { + prompt: string; + negativePrompt?: string; + width: number; + height: number; + /** Number of images, default 1. */ + count?: number; + seed?: number; + /** Source image for image-to-image and inpainting. */ + source?: MediaFile; + /** 0–1: how far the result may move away from the source. */ + strength?: number; + /** Inpainting mask, same size as the source; white areas are repainted. */ + mask?: MediaFile; + /** Workflow template id; the provider's default when omitted. */ + workflow?: string; + signal?: AbortSignal; + onProgress?: MediaProgressListener; +} + +export interface VideoGenerationRequest { + prompt: string; + negativePrompt?: string; + width: number; + height: number; + durationSeconds: number; + fps?: number; + seed?: number; + /** Start frame for image-to-video. */ + source?: MediaFile; + workflow?: string; + signal?: AbortSignal; + onProgress?: MediaProgressListener; +} + +export interface ImageGenerationCapabilities { + textToImage: boolean; + imageToImage: boolean; + inpaint: boolean; + maxWidth: number; + maxHeight: number; +} + +export interface VideoGenerationCapabilities { + textToVideo: boolean; + imageToVideo: boolean; + maxDurationSeconds: number; +} + +export interface ImageGenerationProvider { + describe(): AdapterDescriptor; + generateImage(request: ImageGenerationRequest): Promise; + /** Lets the media runner wait for free GPU memory instead of failing mid-generation. */ + gpuStatus?(signal?: AbortSignal): Promise; +} + +export interface VideoGenerationProvider { + describe(): AdapterDescriptor; + generateVideo(request: VideoGenerationRequest): Promise; + gpuStatus?(signal?: AbortSignal): Promise; +} + +export interface TranscriptSegment { + startMs: number; + endMs: number; + text: string; +} + +export interface Transcript { + text: string; + /** Detected or requested language (BCP 47), when the backend reports it. */ + language?: string; + /** Timed segments; empty when the backend returns plain text only. */ + segments: TranscriptSegment[]; +} + +export interface TranscriptionRequest { + audio: Uint8Array; + mimeType: string; + fileName?: string; + /** Language hint (BCP 47); detected when omitted. */ + language?: string; + signal?: AbortSignal; +} + +export interface TranscriptionCapabilities { + segments: boolean; + languageDetection: boolean; + maxBytes: number; +} + +export interface TranscriptionProvider { + describe(): AdapterDescriptor; + transcribe(request: TranscriptionRequest): Promise; +} + +/** A file an operation produced in its temp directory, to be stored as an artifact. */ +export interface MediaJobOutputFile { + path: string; + name: string; + mimeType: string; + width?: number; + height?: number; + durationMs?: number; + /** Poster frame for a video, as a file in the temp directory. */ + posterPath?: string; + parentArtifactId?: string; + preset?: MediaPresetId; +} + +/** An artifact the job may read, copied into the job's temp directory. */ +export interface MediaJobInputFile { + artifactId: string; + path: string; + name: string; + mimeType: string; + kind: MediaOutputKind; + size: number; + width?: number; + height?: number; + durationMs?: number; +} + +/** What the media runner hands an operation while a job runs. */ +export interface MediaJobContext { + jobId: string; + spaceId: string; + userId: string; + botId: string | null; + /** The input stored with the job; each operation validates it with its own schema. */ + input: unknown; + /** Private scratch directory, removed when the job ends. */ + tempDir: string; + signal: AbortSignal; + progress(fraction: number): Promise; + /** Copies an artifact of the job's space that the job's user may read into tempDir. */ + fetchArtifact(artifactId: string): Promise; + /** Stores a produced file as an artifact and adds it to the media card. */ + addOutput(file: MediaJobOutputFile): Promise; +} + +export interface MediaOperationResult { + /** One or two sentences for the bot that started the job. */ + summary: string; +} + +export interface MediaOperationHandler { + operation: MediaOperation; + run(context: MediaJobContext): Promise; +} + +/** Thrown by an operation that cannot start yet, e.g. while the GPU lacks free memory. */ +export class MediaJobDeferredError extends Error { + constructor( + message: string, + readonly retryAfterMs: number, + ) { + super(message); + this.name = "MediaJobDeferredError"; + } +} + +export interface StartMediaJobInput { + operation: MediaOperation; + /** Validated by the operation's input schema before the job is created. */ + input: unknown; + title: string; + spaceId: string; + userId: string; + botId: string | null; + groupId: string | null; + threadId: string; + runId: string | null; +} + +export interface MediaJobSnapshot { + id: string; + operation: MediaOperation; + status: MediaJobStatus; + title: string; + progress: number | null; + error: string | null; + outputs: MediaOutput[]; +} + +/** Which media operations the space can run right now. */ +export type MediaCapabilities = Record; + +/** + * Starts media jobs and answers which operations are available. Tools call this; the job runs + * in a background worker and keeps its media card current. + */ +export interface MediaRuntime { + capabilities(spaceId: string): Promise; + start(input: StartMediaJobInput): Promise<{ jobId: string; messageId: string }>; + get(jobId: string): Promise; + cancel(jobId: string): Promise; +} diff --git a/packages/adapters/src/thread-artifacts.ts b/packages/adapters/src/thread-artifacts.ts index b9bf27d..72bc432 100644 --- a/packages/adapters/src/thread-artifacts.ts +++ b/packages/adapters/src/thread-artifacts.ts @@ -39,7 +39,10 @@ export async function attachWorkspaceFileToThread( bytes: Uint8Array; operationId: string; }, -): Promise<{ artifactId: string; block: Extract }> { +): Promise<{ + artifactId: string; + block: Extract; +}> { const name = path.basename(input.filePath) || input.filePath; const mimeType = inferAttachmentMimeType(name); if (!mimeType) { diff --git a/packages/contracts/src/attachments.test.ts b/packages/contracts/src/attachments.test.ts index 2b2c873..fb593cc 100644 --- a/packages/contracts/src/attachments.test.ts +++ b/packages/contracts/src/attachments.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { MessageBlock, validateThreadsSendInput } from "./index.js"; +import { attachmentKindOf, MessageBlock, validateThreadsSendInput } from "./index.js"; describe("attachment contracts", () => { it("parses image and file message blocks", () => { @@ -28,4 +28,45 @@ describe("attachment contracts", () => { expect(validateThreadsSendInput({})).toBe(false); expect(validateThreadsSendInput({ artifactIds: ["a", "b", "c", "d", "e"] })).toBe(true); }); + + it("classifies media types and parses media blocks", () => { + expect(attachmentKindOf("image/webp")).toBe("image"); + expect(attachmentKindOf("video/mp4")).toBe("video"); + expect(attachmentKindOf("audio/mpeg")).toBe("audio"); + expect(attachmentKindOf("text/vtt")).toBe("file"); + expect( + MessageBlock.parse({ + kind: "video", + artifactId: "art_3", + mimeType: "video/mp4", + name: "cut.mp4", + size: 2048, + durationMs: 15000, + width: 1080, + height: 1920, + }), + ).toMatchObject({ kind: "video", width: 1080 }); + expect( + MessageBlock.parse({ + kind: "media", + jobId: "job_1", + operation: "edit_video", + status: "running", + title: "Cut to 9:16", + progress: 0.4, + outputs: [], + }), + ).toMatchObject({ kind: "media", status: "running" }); + expect(() => + MessageBlock.parse({ + kind: "media", + jobId: "job_1", + operation: "edit_video", + status: "running", + title: "Cut", + progress: 1.5, + outputs: [], + }), + ).toThrow(); + }); }); diff --git a/packages/contracts/src/attachments.ts b/packages/contracts/src/attachments.ts index 7853fe4..3434e35 100644 --- a/packages/contracts/src/attachments.ts +++ b/packages/contracts/src/attachments.ts @@ -10,25 +10,67 @@ export const ATTACHMENT_IMAGE_MIME_TYPES = [ "image/gif", ] as const; +export const ATTACHMENT_VIDEO_MIME_TYPES = ["video/mp4", "video/webm", "video/quicktime"] as const; + +export const ATTACHMENT_AUDIO_MIME_TYPES = [ + "audio/mpeg", + "audio/mp4", + "audio/wav", + "audio/webm", + "audio/ogg", +] as const; + export const ATTACHMENT_FILE_MIME_TYPES = [ "application/pdf", "text/plain", "text/markdown", "text/csv", "application/json", + "text/vtt", + "application/x-subrip", ] as const; export const ATTACHMENT_ALLOWED_MIME_TYPES = [ ...ATTACHMENT_IMAGE_MIME_TYPES, + ...ATTACHMENT_VIDEO_MIME_TYPES, + ...ATTACHMENT_AUDIO_MIME_TYPES, ...ATTACHMENT_FILE_MIME_TYPES, ] as const; +export type AttachmentKind = "image" | "video" | "audio" | "file"; + +/** + * Size caps for artifacts that do not travel inline as base64 (streamed uploads, generated and + * edited media). Inline RPC uploads stay capped by ATTACHMENT_MAX_BYTES. + */ +export const ARTIFACT_MAX_BYTES_BY_KIND: Record = { + image: 25 * 1024 * 1024, + video: 500 * 1024 * 1024, + audio: 200 * 1024 * 1024, + file: 25 * 1024 * 1024, +}; + export type AttachmentMimeType = (typeof ATTACHMENT_ALLOWED_MIME_TYPES)[number]; export function isAttachmentImageMimeType(mimeType: string): boolean { return (ATTACHMENT_IMAGE_MIME_TYPES as readonly string[]).includes(mimeType); } +export function isAttachmentVideoMimeType(mimeType: string): boolean { + return (ATTACHMENT_VIDEO_MIME_TYPES as readonly string[]).includes(mimeType); +} + +export function isAttachmentAudioMimeType(mimeType: string): boolean { + return (ATTACHMENT_AUDIO_MIME_TYPES as readonly string[]).includes(mimeType); +} + +export function attachmentKindOf(mimeType: string): AttachmentKind { + if (isAttachmentImageMimeType(mimeType)) return "image"; + if (isAttachmentVideoMimeType(mimeType)) return "video"; + if (isAttachmentAudioMimeType(mimeType)) return "audio"; + return "file"; +} + export function isAllowedAttachmentMimeType(mimeType: string): mimeType is AttachmentMimeType { return (ATTACHMENT_ALLOWED_MIME_TYPES as readonly string[]).includes(mimeType); } diff --git a/packages/contracts/src/events.ts b/packages/contracts/src/events.ts index c15d8d0..ccf3c94 100644 --- a/packages/contracts/src/events.ts +++ b/packages/contracts/src/events.ts @@ -2,6 +2,7 @@ import * as z from "zod"; import { BotSecretDestination } from "./bot-secrets.js"; import { Id } from "./ids.js"; import { McpTransportSchema } from "./mcp.js"; +import { AudioBlockSchema, MediaJobBlockSchema, VideoBlockSchema } from "./media.js"; export const ProductEventType = z.enum([ "thread.message.created", @@ -248,6 +249,9 @@ export const MessageBlock = z.discriminatedUnion("kind", [ name: z.string(), size: z.number().int().nonnegative(), }), + VideoBlockSchema, + AudioBlockSchema, + MediaJobBlockSchema, z.object({ kind: z.literal("handoff"), fromBotId: Id, diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 3865e1b..ba2c70b 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -9,6 +9,7 @@ export * from "./inbox.js"; export * from "./integration-settings.js"; export * from "./local-settings.js"; export * from "./mcp.js"; +export * from "./media.js"; export * from "./model-tiers.js"; export * from "./openai-compatible-ui.js"; export * from "./project-goals.js"; diff --git a/packages/contracts/src/media.ts b/packages/contracts/src/media.ts new file mode 100644 index 0000000..3654e9e --- /dev/null +++ b/packages/contracts/src/media.ts @@ -0,0 +1,113 @@ +import * as z from "zod"; +import { Id } from "./ids.js"; + +/** What a media job does. Generation runs on the configured image/video backend, edits run locally. */ +export const MediaOperationSchema = z.enum([ + "generate_image", + "generate_video", + "edit_image", + "edit_video", + "export_video", + "transcribe", +]); +export type MediaOperation = z.infer; + +/** `waiting` means the job is queued behind another GPU job or for free GPU memory. */ +export const MediaJobStatusSchema = z.enum([ + "queued", + "waiting", + "running", + "done", + "failed", + "cancelled", +]); +export type MediaJobStatus = z.infer; + +export const MEDIA_JOB_ACTIVE_STATUSES = [ + "queued", + "waiting", + "running", +] as const satisfies readonly MediaJobStatus[]; + +export const MediaOutputKindSchema = z.enum(["image", "video", "audio", "file"]); +export type MediaOutputKind = z.infer; + +/** + * Output formats for video cuts and exports. Width and height are the encoded frame; every + * preset encodes H.264 + AAC in MP4 so the result plays everywhere it is posted. + */ +export const MEDIA_PRESETS = { + vertical: { aspect: "9:16", width: 1080, height: 1920 }, + landscape: { aspect: "16:9", width: 1920, height: 1080 }, + square: { aspect: "1:1", width: 1080, height: 1080 }, + portrait: { aspect: "4:5", width: 1080, height: 1350 }, + classic: { aspect: "4:3", width: 1440, height: 1080 }, + cinema: { aspect: "21:9", width: 2560, height: 1080 }, +} as const; +export const MediaPresetIdSchema = z.enum([ + "vertical", + "landscape", + "square", + "portrait", + "classic", + "cinema", +]); +export type MediaPresetId = z.infer; +export type MediaPreset = (typeof MEDIA_PRESETS)[MediaPresetId]; + +/** How a frame of another aspect ratio fills a preset. */ +export const MediaFitSchema = z.enum(["crop", "pad", "blur"]); +export type MediaFit = z.infer; + +export const MediaOutputSchema = z.object({ + artifactId: Id, + kind: MediaOutputKindSchema, + mimeType: z.string(), + name: z.string(), + size: z.number().int().nonnegative(), + width: z.number().int().positive().optional(), + height: z.number().int().positive().optional(), + durationMs: z.number().int().nonnegative().optional(), + posterArtifactId: Id.optional(), + /** The artifact this output was made from, for version history. */ + parentArtifactId: Id.optional(), + preset: MediaPresetIdSchema.optional(), +}); +export type MediaOutput = z.infer; + +/** The live media card a job keeps up to date in the conversation. */ +export const MediaJobBlockSchema = z.object({ + kind: z.literal("media"), + jobId: Id, + operation: MediaOperationSchema, + status: MediaJobStatusSchema, + /** Short label: the prompt, or what the edit does. */ + title: z.string(), + progress: z.number().min(0).max(1).optional(), + error: z.string().optional(), + outputs: z.array(MediaOutputSchema), +}); +export type MediaJobBlock = z.infer; + +export const VideoBlockSchema = z.object({ + kind: z.literal("video"), + artifactId: Id, + mimeType: z.string(), + name: z.string(), + size: z.number().int().nonnegative(), + durationMs: z.number().int().nonnegative().optional(), + width: z.number().int().positive().optional(), + height: z.number().int().positive().optional(), + posterArtifactId: Id.optional(), +}); +export type VideoBlock = z.infer; + +export const AudioBlockSchema = z.object({ + kind: z.literal("audio"), + artifactId: Id, + mimeType: z.string(), + name: z.string(), + size: z.number().int().nonnegative(), + durationMs: z.number().int().nonnegative().optional(), +}); +export type AudioBlock = z.infer; diff --git a/packages/core/src/attachments.test.ts b/packages/core/src/attachments.test.ts index 0a3d661..6db2e57 100644 --- a/packages/core/src/attachments.test.ts +++ b/packages/core/src/attachments.test.ts @@ -5,6 +5,7 @@ import { blocksToAgentHistoryText, decodeAttachmentBase64, inferAttachmentMimeType, + messageBlockForArtifact, promptTextForAttachments, userTurnBlocksForRun, validateAttachmentMimeType, @@ -108,4 +109,61 @@ describe("peer message history", () => { ]), ).toBe("[to Analyst] chart it"); }); + + it("builds video and audio blocks and keeps media artifact ids in history", () => { + const video = messageBlockForArtifact({ + id: "art_v", + name: "clip.mp4", + mimeType: "video/mp4", + size: 10, + width: 1920, + height: 1080, + durationMs: 3000, + }); + expect(video).toEqual({ + kind: "video", + artifactId: "art_v", + mimeType: "video/mp4", + name: "clip.mp4", + size: 10, + durationMs: 3000, + width: 1920, + height: 1080, + }); + expect( + messageBlockForArtifact({ id: "art_a", name: "a.mp3", mimeType: "audio/mpeg", size: 5 }), + ).toEqual({ + kind: "audio", + artifactId: "art_a", + mimeType: "audio/mpeg", + name: "a.mp3", + size: 5, + }); + expect(inferAttachmentMimeType("talk.MOV")).toBe("video/quicktime"); + expect(inferAttachmentMimeType("subs.srt")).toBe("application/x-subrip"); + expect( + blocksToAgentHistoryText([ + video, + { + kind: "media", + jobId: "job_1", + operation: "edit_video", + status: "done", + title: "Cut to 9:16", + outputs: [ + { + artifactId: "art_out", + kind: "video", + mimeType: "video/mp4", + name: "clip-vertical.mp4", + size: 9, + preset: "vertical", + }, + ], + }, + ]), + ).toBe( + "[video: clip.mp4 (artifact art_v)]\n[media edit_video done: Cut to 9:16 -> video clip-vertical.mp4 (artifact art_out)]", + ); + }); }); diff --git a/packages/core/src/attachments.ts b/packages/core/src/attachments.ts index 9afe692..b4a6759 100644 --- a/packages/core/src/attachments.ts +++ b/packages/core/src/attachments.ts @@ -1,7 +1,9 @@ import { ATTACHMENT_MAX_BASE64_LENGTH, ATTACHMENT_MAX_BYTES, + type AttachmentKind, type AttachmentMimeType, + attachmentKindOf, isAllowedAttachmentMimeType, isAttachmentImageMimeType, type MessageBlock, @@ -45,8 +47,8 @@ export function validateAttachmentMimeType(mimeType: string): void { } } -export function attachmentKindForMimeType(mimeType: string): "image" | "file" { - return isAttachmentImageMimeType(mimeType) ? "image" : "file"; +export function attachmentKindForMimeType(mimeType: string): AttachmentKind { + return attachmentKindOf(mimeType); } export function messageBlockForArtifact(artifact: { @@ -54,8 +56,12 @@ export function messageBlockForArtifact(artifact: { name: string; mimeType: string; size: number; -}): Extract { - if (attachmentKindForMimeType(artifact.mimeType) === "image") { + width?: number | null; + height?: number | null; + durationMs?: number | null; +}): Extract { + const kind = attachmentKindForMimeType(artifact.mimeType); + if (kind === "image") { return { kind: "image", artifactId: artifact.id, @@ -63,6 +69,28 @@ export function messageBlockForArtifact(artifact: { name: artifact.name, }; } + if (kind === "video") { + return { + kind: "video", + artifactId: artifact.id, + mimeType: artifact.mimeType, + name: artifact.name, + size: artifact.size, + ...(artifact.durationMs != null ? { durationMs: artifact.durationMs } : {}), + ...(artifact.width != null ? { width: artifact.width } : {}), + ...(artifact.height != null ? { height: artifact.height } : {}), + }; + } + if (kind === "audio") { + return { + kind: "audio", + artifactId: artifact.id, + mimeType: artifact.mimeType, + name: artifact.name, + size: artifact.size, + ...(artifact.durationMs != null ? { durationMs: artifact.durationMs } : {}), + }; + } return { kind: "file", artifactId: artifact.id, @@ -95,6 +123,16 @@ export function blocksToAgentHistoryText(blocks: MessageBlock[]): string { if (block.kind === "file") { return `[file: ${block.name} (${block.mimeType}, ${block.size} bytes)]`; } + // Media keeps the artifact id so a later turn can edit or export what it refers to. + if (block.kind === "video" || block.kind === "audio") { + return `[${block.kind}: ${block.name} (artifact ${block.artifactId})]`; + } + if (block.kind === "media") { + const outputs = block.outputs + .map((output) => `${output.kind} ${output.name} (artifact ${output.artifactId})`) + .join(", "); + return `[media ${block.operation} ${block.status}: ${block.title}${outputs ? ` -> ${outputs}` : ""}]`; + } // Keep attribution on peer messages: without it a later turn cannot tell // which lines came from another bot rather than the user. if (block.kind === "bot_message_received") return `[from ${block.fromBotName}] ${block.text}`; @@ -124,6 +162,17 @@ const EXTENSION_MIME_TYPES: Record = { ".markdown": "text/markdown", ".csv": "text/csv", ".json": "application/json", + ".mp4": "video/mp4", + ".m4v": "video/mp4", + ".webm": "video/webm", + ".mov": "video/quicktime", + ".mp3": "audio/mpeg", + ".m4a": "audio/mp4", + ".wav": "audio/wav", + ".ogg": "audio/ogg", + ".oga": "audio/ogg", + ".vtt": "text/vtt", + ".srt": "application/x-subrip", }; const MIME_TYPE_EXTENSIONS: Record = { @@ -136,6 +185,16 @@ const MIME_TYPE_EXTENSIONS: Record = { "text/markdown": ".md", "text/csv": ".csv", "application/json": ".json", + "video/mp4": ".mp4", + "video/webm": ".webm", + "video/quicktime": ".mov", + "audio/mpeg": ".mp3", + "audio/mp4": ".m4a", + "audio/wav": ".wav", + "audio/webm": ".weba", + "audio/ogg": ".ogg", + "text/vtt": ".vtt", + "application/x-subrip": ".srt", }; export function inferAttachmentMimeType( diff --git a/packages/db/prisma/migrations/20261003130000_media_artifacts_and_jobs/migration.sql b/packages/db/prisma/migrations/20261003130000_media_artifacts_and_jobs/migration.sql new file mode 100644 index 0000000..f1f59c0 --- /dev/null +++ b/packages/db/prisma/migrations/20261003130000_media_artifacts_and_jobs/migration.sql @@ -0,0 +1,57 @@ +-- Media: artifacts remember their pixel size, play length and the artifact they were made from, +-- and generations, edits and transcriptions run as media jobs outside the agent turn. +ALTER TABLE "artifacts" ADD COLUMN "width" INTEGER; +ALTER TABLE "artifacts" ADD COLUMN "height" INTEGER; +ALTER TABLE "artifacts" ADD COLUMN "durationMs" INTEGER; +ALTER TABLE "artifacts" ADD COLUMN "parentArtifactId" TEXT; + +CREATE INDEX "artifacts_parentArtifactId_idx" ON "artifacts"("parentArtifactId"); + +ALTER TABLE "artifacts" ADD CONSTRAINT "artifacts_parentArtifactId_fkey" + FOREIGN KEY ("parentArtifactId") REFERENCES "artifacts"("id") ON DELETE SET NULL ON UPDATE CASCADE + NOT VALID; +ALTER TABLE "artifacts" ADD CONSTRAINT "artifacts_media_metadata_chk" + CHECK ( + ("width" IS NULL OR "width" > 0) + AND ("height" IS NULL OR "height" > 0) + AND ("durationMs" IS NULL OR "durationMs" >= 0) + ) NOT VALID; + +CREATE TABLE "media_jobs" ( + "id" TEXT NOT NULL, + "spaceId" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "botId" TEXT, + "groupId" TEXT, + "threadId" TEXT NOT NULL, + "messageId" TEXT, + "runId" TEXT, + "operation" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'queued', + "title" TEXT NOT NULL DEFAULT '', + "input" JSONB NOT NULL, + "progress" DOUBLE PRECISION, + "error" TEXT, + "outputArtifactIds" TEXT[] DEFAULT ARRAY[]::TEXT[], + "attempts" INTEGER NOT NULL DEFAULT 0, + "cancelRequested" BOOLEAN NOT NULL DEFAULT false, + "startedAt" TIMESTAMP(3), + "finishedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "media_jobs_pkey" PRIMARY KEY ("id"), + CONSTRAINT "media_jobs_operation_chk" CHECK ("operation" IN ( + 'generate_image', 'generate_video', 'edit_image', 'edit_video', 'export_video', 'transcribe' + )), + CONSTRAINT "media_jobs_status_chk" CHECK ("status" IN ( + 'queued', 'waiting', 'running', 'done', 'failed', 'cancelled' + )), + CONSTRAINT "media_jobs_progress_chk" CHECK ("progress" IS NULL OR ("progress" >= 0 AND "progress" <= 1)) +); + +CREATE INDEX "media_jobs_spaceId_status_idx" ON "media_jobs"("spaceId", "status"); +CREATE INDEX "media_jobs_threadId_idx" ON "media_jobs"("threadId"); + +ALTER TABLE "media_jobs" ADD CONSTRAINT "media_jobs_spaceId_fkey" + FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20261003130001_validate_media_artifacts_and_jobs_constraints/migration.sql b/packages/db/prisma/migrations/20261003130001_validate_media_artifacts_and_jobs_constraints/migration.sql new file mode 100644 index 0000000..5a95b6e --- /dev/null +++ b/packages/db/prisma/migrations/20261003130001_validate_media_artifacts_and_jobs_constraints/migration.sql @@ -0,0 +1,4 @@ +-- Kept separate from creation so PostgreSQL validates existing rows with a SHARE UPDATE +-- EXCLUSIVE lock (reads and writes continue) after the NOT VALID constraints have committed. +ALTER TABLE "artifacts" VALIDATE CONSTRAINT "artifacts_parentArtifactId_fkey"; +ALTER TABLE "artifacts" VALIDATE CONSTRAINT "artifacts_media_metadata_chk"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index a6dc085..b9773da 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -197,6 +197,7 @@ model Space { agentHomes AgentHome[] browserProfiles BrowserProfile[] artifacts Artifact[] + mediaJobs MediaJob[] usageRecords UsageRecord[] notificationPreferences NotificationPreference[] computers Computer[] @@ -1065,13 +1066,54 @@ model Artifact { size Int hash String storageKey String + /// Pixel size of images and videos and play length of audio and video, when known. + width Int? + height Int? + durationMs Int? + /// The artifact this one was made from (an edit, cut or export), for version history. + parentArtifactId String? + parent Artifact? @relation("ArtifactVersions", fields: [parentArtifactId], references: [id], onDelete: SetNull) + versions Artifact[] @relation("ArtifactVersions") createdAt DateTime @default(now()) @@index([spaceId, botId]) @@index([groupId]) + @@index([parentArtifactId]) @@map("artifacts") } +/// A generation, edit or transcription that runs outside the agent turn and reports into a +/// media card in the conversation. Like CloudAgent, provenance ids carry no foreign keys. +model MediaJob { + id String @id @default(cuid()) + spaceId String + space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade) + userId String + botId String? + groupId String? + threadId String + /// The bot message that carries the media card. + messageId String? + runId String? + operation String + status String @default("queued") + title String @default("") + input Json + progress Float? + error String? + outputArtifactIds String[] @default([]) + attempts Int @default(0) + cancelRequested Boolean @default(false) + startedAt DateTime? + finishedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([spaceId, status]) + @@index([threadId]) + @@map("media_jobs") +} + model UsageRecord { id String @id @default(cuid()) spaceId String