cynarAI/evOo-local
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat(media): add media contracts, schema and provider interfaces

Browse Source 
Groundwork for image and video generation, video editing and transcription.
Nothing calls it yet; the media runner, providers, tools and UI build on it.

- Contracts: video, audio and subtitle MIME types, per-kind size caps for
  streamed and generated artifacts, video/audio message blocks and the live
  media card block, media operations, job statuses and output presets
  (9:16, 16:9, 1:1, 4:5, 4:3, 21:9).
- Core: attachment kinds include video and audio, artifacts become video or
  audio blocks, and agent history keeps media artifact ids so later turns
  can refer to them.
- Schema: artifacts record width, height, duration and the artifact they
  were made from; media_jobs tracks generations, edits and transcriptions.
- Adapter kit: provider-neutral image generation, video generation and
  transcription interfaces, the media job context operations run in, and
  the MediaRuntime that tools use to start jobs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude 2026-10-03 13:04:16 +00:00
parent 3d180dc2dd
commit 7ff93f08bb
No known key found for this signature in database
13 changed files with 660 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/adapter-kit/src/index.ts
View File

@ -1,5 +1,6 @@
export * from "./background-jobs.js";
export * from "./cloud-agents.js";
export * from "./interfaces.js";
export * from "./media.js";
export * from "./registry.js";
export * from "./types.js";

229
packages/adapter-kit/src/media.ts Normal file
View File

@ -0,0 +1,229 @@
import type {
MediaJobStatus,
MediaOperation,
MediaOutput,
MediaOutputKind,
MediaPresetId,
} from "@rakazo/contracts";
import type { AdapterDescriptor } from "./types.js";
/** Bytes handed to or returned by a media backend. */
export interface MediaFile {
bytes: Uint8Array;
mimeType: string;
name?: string;
}
export interface GeneratedMedia extends MediaFile {
width?: number;
height?: number;
durationMs?: number;
}
/** Reports progress as a fraction from 0 to 1. */
export type MediaProgressListener = (fraction: number) => void;
/** Free and total memory of the backend's GPU, when the backend can tell. */
export interface GpuStatus {
vramFreeMb: number;
vramTotalMb: number;
}
export interface ImageGenerationRequest {
prompt: string;
negativePrompt?: string;
width: number;
height: number;
/** Number of images, default 1. */
count?: number;
seed?: number;
/** Source image for image-to-image and inpainting. */
source?: MediaFile;
/** 0–1: how far the result may move away from the source. */
strength?: number;
/** Inpainting mask, same size as the source; white areas are repainted. */
mask?: MediaFile;
/** Workflow template id; the provider's default when omitted. */
workflow?: string;
signal?: AbortSignal;
onProgress?: MediaProgressListener;
}
export interface VideoGenerationRequest {
prompt: string;
negativePrompt?: string;
width: number;
height: number;
durationSeconds: number;
fps?: number;
seed?: number;
/** Start frame for image-to-video. */
source?: MediaFile;
workflow?: string;
signal?: AbortSignal;
onProgress?: MediaProgressListener;
}
export interface ImageGenerationCapabilities {
textToImage: boolean;
imageToImage: boolean;
inpaint: boolean;
maxWidth: number;
maxHeight: number;
}
export interface VideoGenerationCapabilities {
textToVideo: boolean;
imageToVideo: boolean;
maxDurationSeconds: number;
}
export interface ImageGenerationProvider {
describe(): AdapterDescriptor<ImageGenerationCapabilities>;
generateImage(request: ImageGenerationRequest): Promise<GeneratedMedia[]>;
/** Lets the media runner wait for free GPU memory instead of failing mid-generation. */
gpuStatus?(signal?: AbortSignal): Promise<GpuStatus | undefined>;
}
export interface VideoGenerationProvider {
describe(): AdapterDescriptor<VideoGenerationCapabilities>;
generateVideo(request: VideoGenerationRequest): Promise<GeneratedMedia[]>;
gpuStatus?(signal?: AbortSignal): Promise<GpuStatus | undefined>;
}
export interface TranscriptSegment {
startMs: number;
endMs: number;
text: string;
}
export interface Transcript {
text: string;
/** Detected or requested language (BCP 47), when the backend reports it. */
language?: string;
/** Timed segments; empty when the backend returns plain text only. */
segments: TranscriptSegment[];
}
export interface TranscriptionRequest {
audio: Uint8Array;
mimeType: string;
fileName?: string;
/** Language hint (BCP 47); detected when omitted. */
language?: string;
signal?: AbortSignal;
}
export interface TranscriptionCapabilities {
segments: boolean;
languageDetection: boolean;
maxBytes: number;
}
export interface TranscriptionProvider {
describe(): AdapterDescriptor<TranscriptionCapabilities>;
transcribe(request: TranscriptionRequest): Promise<Transcript>;
}
/** A file an operation produced in its temp directory, to be stored as an artifact. */
export interface MediaJobOutputFile {
path: string;
name: string;
mimeType: string;
width?: number;
height?: number;
durationMs?: number;
/** Poster frame for a video, as a file in the temp directory. */
posterPath?: string;
parentArtifactId?: string;
preset?: MediaPresetId;
}
/** An artifact the job may read, copied into the job's temp directory. */
export interface MediaJobInputFile {
artifactId: string;
path: string;
name: string;
mimeType: string;
kind: MediaOutputKind;
size: number;
width?: number;
height?: number;
durationMs?: number;
}
/** What the media runner hands an operation while a job runs. */
export interface MediaJobContext {
jobId: string;
spaceId: string;
userId: string;
botId: string | null;
/** The input stored with the job; each operation validates it with its own schema. */
input: unknown;
/** Private scratch directory, removed when the job ends. */
tempDir: string;
signal: AbortSignal;
progress(fraction: number): Promise<void>;
/** Copies an artifact of the job's space that the job's user may read into tempDir. */
fetchArtifact(artifactId: string): Promise<MediaJobInputFile>;
/** Stores a produced file as an artifact and adds it to the media card. */
addOutput(file: MediaJobOutputFile): Promise<MediaOutput>;
}
export interface MediaOperationResult {
/** One or two sentences for the bot that started the job. */
summary: string;
}
export interface MediaOperationHandler {
operation: MediaOperation;
run(context: MediaJobContext): Promise<MediaOperationResult>;
}
/** Thrown by an operation that cannot start yet, e.g. while the GPU lacks free memory. */
export class MediaJobDeferredError extends Error {
constructor(
message: string,
readonly retryAfterMs: number,
) {
super(message);
this.name = "MediaJobDeferredError";
}
}
export interface StartMediaJobInput {
operation: MediaOperation;
/** Validated by the operation's input schema before the job is created. */
input: unknown;
title: string;
spaceId: string;
userId: string;
botId: string | null;
groupId: string | null;
threadId: string;
runId: string | null;
}
export interface MediaJobSnapshot {
id: string;
operation: MediaOperation;
status: MediaJobStatus;
title: string;
progress: number | null;
error: string | null;
outputs: MediaOutput[];
}
/** Which media operations the space can run right now. */
export type MediaCapabilities = Record<MediaOperation, boolean>;
/**
* Starts media jobs and answers which operations are available. Tools call this; the job runs
* in a background worker and keeps its media card current.
*/
export interface MediaRuntime {
capabilities(spaceId: string): Promise<MediaCapabilities>;
start(input: StartMediaJobInput): Promise<{ jobId: string; messageId: string }>;
get(jobId: string): Promise<MediaJobSnapshot | null>;
cancel(jobId: string): Promise<void>;
}

5
packages/adapters/src/thread-artifacts.ts
View File

@ -39,7 +39,10 @@ export async function attachWorkspaceFileToThread(
bytes: Uint8Array;
operationId: string;
},
): Promise<{ artifactId: string; block: Extract<MessageBlock, { kind: "image" | "file" }> }> {
): Promise<{
artifactId: string;
block: Extract<MessageBlock, { kind: "image" | "video" | "audio" | "file" }>;
}> {
const name = path.basename(input.filePath) || input.filePath;
const mimeType = inferAttachmentMimeType(name);
if (!mimeType) {

43
packages/contracts/src/attachments.test.ts
View File

@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { MessageBlock, validateThreadsSendInput } from "./index.js";
import { attachmentKindOf, MessageBlock, validateThreadsSendInput } from "./index.js";
describe("attachment contracts", () => {
it("parses image and file message blocks", () => {
@ -28,4 +28,45 @@ describe("attachment contracts", () => {
expect(validateThreadsSendInput({})).toBe(false);
expect(validateThreadsSendInput({ artifactIds: ["a", "b", "c", "d", "e"] })).toBe(true);
});
it("classifies media types and parses media blocks", () => {
expect(attachmentKindOf("image/webp")).toBe("image");
expect(attachmentKindOf("video/mp4")).toBe("video");
expect(attachmentKindOf("audio/mpeg")).toBe("audio");
expect(attachmentKindOf("text/vtt")).toBe("file");
expect(
MessageBlock.parse({
kind: "video",
artifactId: "art_3",
mimeType: "video/mp4",
name: "cut.mp4",
size: 2048,
durationMs: 15000,
width: 1080,
height: 1920,
}),
).toMatchObject({ kind: "video", width: 1080 });
expect(
MessageBlock.parse({
kind: "media",
jobId: "job_1",
operation: "edit_video",
status: "running",
title: "Cut to 9:16",
progress: 0.4,
outputs: [],
}),
).toMatchObject({ kind: "media", status: "running" });
expect(() =>
MessageBlock.parse({
kind: "media",
jobId: "job_1",
operation: "edit_video",
status: "running",
title: "Cut",
progress: 1.5,
outputs: [],
}),
).toThrow();
});
});

42
packages/contracts/src/attachments.ts
View File

@ -10,25 +10,67 @@ export const ATTACHMENT_IMAGE_MIME_TYPES = [
"image/gif",
] as const;
export const ATTACHMENT_VIDEO_MIME_TYPES = ["video/mp4", "video/webm", "video/quicktime"] as const;
export const ATTACHMENT_AUDIO_MIME_TYPES = [
"audio/mpeg",
"audio/mp4",
"audio/wav",
"audio/webm",
"audio/ogg",
] as const;
export const ATTACHMENT_FILE_MIME_TYPES = [
"application/pdf",
"text/plain",
"text/markdown",
"text/csv",
"application/json",
"text/vtt",
"application/x-subrip",
] as const;
export const ATTACHMENT_ALLOWED_MIME_TYPES = [
...ATTACHMENT_IMAGE_MIME_TYPES,
...ATTACHMENT_VIDEO_MIME_TYPES,
...ATTACHMENT_AUDIO_MIME_TYPES,
...ATTACHMENT_FILE_MIME_TYPES,
] as const;
export type AttachmentKind = "image" | "video" | "audio" | "file";
/**
* Size caps for artifacts that do not travel inline as base64 (streamed uploads, generated and
* edited media). Inline RPC uploads stay capped by ATTACHMENT_MAX_BYTES.
*/
export const ARTIFACT_MAX_BYTES_BY_KIND: Record<AttachmentKind, number> = {
image: 25 * 1024 * 1024,
video: 500 * 1024 * 1024,
audio: 200 * 1024 * 1024,
file: 25 * 1024 * 1024,
};
export type AttachmentMimeType = (typeof ATTACHMENT_ALLOWED_MIME_TYPES)[number];
export function isAttachmentImageMimeType(mimeType: string): boolean {
return (ATTACHMENT_IMAGE_MIME_TYPES as readonly string[]).includes(mimeType);
}
export function isAttachmentVideoMimeType(mimeType: string): boolean {
return (ATTACHMENT_VIDEO_MIME_TYPES as readonly string[]).includes(mimeType);
}
export function isAttachmentAudioMimeType(mimeType: string): boolean {
return (ATTACHMENT_AUDIO_MIME_TYPES as readonly string[]).includes(mimeType);
}
export function attachmentKindOf(mimeType: string): AttachmentKind {
if (isAttachmentImageMimeType(mimeType)) return "image";
if (isAttachmentVideoMimeType(mimeType)) return "video";
if (isAttachmentAudioMimeType(mimeType)) return "audio";
return "file";
}
export function isAllowedAttachmentMimeType(mimeType: string): mimeType is AttachmentMimeType {
return (ATTACHMENT_ALLOWED_MIME_TYPES as readonly string[]).includes(mimeType);
}

4
packages/contracts/src/events.ts
View File

@ -2,6 +2,7 @@ import * as z from "zod";
import { BotSecretDestination } from "./bot-secrets.js";
import { Id } from "./ids.js";
import { McpTransportSchema } from "./mcp.js";
import { AudioBlockSchema, MediaJobBlockSchema, VideoBlockSchema } from "./media.js";
export const ProductEventType = z.enum([
"thread.message.created",
@ -248,6 +249,9 @@ export const MessageBlock = z.discriminatedUnion("kind", [
name: z.string(),
size: z.number().int().nonnegative(),
}),
VideoBlockSchema,
AudioBlockSchema,
MediaJobBlockSchema,
z.object({
kind: z.literal("handoff"),
fromBotId: Id,

1
packages/contracts/src/index.ts
View File

@ -9,6 +9,7 @@ export * from "./inbox.js";
export * from "./integration-settings.js";
export * from "./local-settings.js";
export * from "./mcp.js";
export * from "./media.js";
export * from "./model-tiers.js";
export * from "./openai-compatible-ui.js";
export * from "./project-goals.js";

113
packages/contracts/src/media.ts Normal file
View File

@ -0,0 +1,113 @@
import * as z from "zod";
import { Id } from "./ids.js";
/** What a media job does. Generation runs on the configured image/video backend, edits run locally. */
export const MediaOperationSchema = z.enum([
"generate_image",
"generate_video",
"edit_image",
"edit_video",
"export_video",
"transcribe",
]);
export type MediaOperation = z.infer<typeof MediaOperationSchema>;
/** `waiting` means the job is queued behind another GPU job or for free GPU memory. */
export const MediaJobStatusSchema = z.enum([
"queued",
"waiting",
"running",
"done",
"failed",
"cancelled",
]);
export type MediaJobStatus = z.infer<typeof MediaJobStatusSchema>;
export const MEDIA_JOB_ACTIVE_STATUSES = [
"queued",
"waiting",
"running",
] as const satisfies readonly MediaJobStatus[];
export const MediaOutputKindSchema = z.enum(["image", "video", "audio", "file"]);
export type MediaOutputKind = z.infer<typeof MediaOutputKindSchema>;
/**
* Output formats for video cuts and exports. Width and height are the encoded frame; every
* preset encodes H.264 + AAC in MP4 so the result plays everywhere it is posted.
*/
export const MEDIA_PRESETS = {
vertical: { aspect: "9:16", width: 1080, height: 1920 },
landscape: { aspect: "16:9", width: 1920, height: 1080 },
square: { aspect: "1:1", width: 1080, height: 1080 },
portrait: { aspect: "4:5", width: 1080, height: 1350 },
classic: { aspect: "4:3", width: 1440, height: 1080 },
cinema: { aspect: "21:9", width: 2560, height: 1080 },
} as const;
export const MediaPresetIdSchema = z.enum([
"vertical",
"landscape",
"square",
"portrait",
"classic",
"cinema",
]);
export type MediaPresetId = z.infer<typeof MediaPresetIdSchema>;
export type MediaPreset = (typeof MEDIA_PRESETS)[MediaPresetId];
/** How a frame of another aspect ratio fills a preset. */
export const MediaFitSchema = z.enum(["crop", "pad", "blur"]);
export type MediaFit = z.infer<typeof MediaFitSchema>;
export const MediaOutputSchema = z.object({
artifactId: Id,
kind: MediaOutputKindSchema,
mimeType: z.string(),
name: z.string(),
size: z.number().int().nonnegative(),
width: z.number().int().positive().optional(),
height: z.number().int().positive().optional(),
durationMs: z.number().int().nonnegative().optional(),
posterArtifactId: Id.optional(),
/** The artifact this output was made from, for version history. */
parentArtifactId: Id.optional(),
preset: MediaPresetIdSchema.optional(),
});
export type MediaOutput = z.infer<typeof MediaOutputSchema>;
/** The live media card a job keeps up to date in the conversation. */
export const MediaJobBlockSchema = z.object({
kind: z.literal("media"),
jobId: Id,
operation: MediaOperationSchema,
status: MediaJobStatusSchema,
/** Short label: the prompt, or what the edit does. */
title: z.string(),
progress: z.number().min(0).max(1).optional(),
error: z.string().optional(),
outputs: z.array(MediaOutputSchema),
});
export type MediaJobBlock = z.infer<typeof MediaJobBlockSchema>;
export const VideoBlockSchema = z.object({
kind: z.literal("video"),
artifactId: Id,
mimeType: z.string(),
name: z.string(),
size: z.number().int().nonnegative(),
durationMs: z.number().int().nonnegative().optional(),
width: z.number().int().positive().optional(),
height: z.number().int().positive().optional(),
posterArtifactId: Id.optional(),
});
export type VideoBlock = z.infer<typeof VideoBlockSchema>;
export const AudioBlockSchema = z.object({
kind: z.literal("audio"),
artifactId: Id,
mimeType: z.string(),
name: z.string(),
size: z.number().int().nonnegative(),
durationMs: z.number().int().nonnegative().optional(),
});
export type AudioBlock = z.infer<typeof AudioBlockSchema>;

58
packages/core/src/attachments.test.ts
View File

@ -5,6 +5,7 @@ import {
blocksToAgentHistoryText,
decodeAttachmentBase64,
inferAttachmentMimeType,
messageBlockForArtifact,
promptTextForAttachments,
userTurnBlocksForRun,
validateAttachmentMimeType,
@ -108,4 +109,61 @@ describe("peer message history", () => {
]),
).toBe("[to Analyst] chart it");
});
it("builds video and audio blocks and keeps media artifact ids in history", () => {
const video = messageBlockForArtifact({
id: "art_v",
name: "clip.mp4",
mimeType: "video/mp4",
size: 10,
width: 1920,
height: 1080,
durationMs: 3000,
});
expect(video).toEqual({
kind: "video",
artifactId: "art_v",
mimeType: "video/mp4",
name: "clip.mp4",
size: 10,
durationMs: 3000,
width: 1920,
height: 1080,
});
expect(
messageBlockForArtifact({ id: "art_a", name: "a.mp3", mimeType: "audio/mpeg", size: 5 }),
).toEqual({
kind: "audio",
artifactId: "art_a",
mimeType: "audio/mpeg",
name: "a.mp3",
size: 5,
});
expect(inferAttachmentMimeType("talk.MOV")).toBe("video/quicktime");
expect(inferAttachmentMimeType("subs.srt")).toBe("application/x-subrip");
expect(
blocksToAgentHistoryText([
video,
{
kind: "media",
jobId: "job_1",
operation: "edit_video",
status: "done",
title: "Cut to 9:16",
outputs: [
{
artifactId: "art_out",
kind: "video",
mimeType: "video/mp4",
name: "clip-vertical.mp4",
size: 9,
preset: "vertical",
},
],
},
]),
).toBe(
"[video: clip.mp4 (artifact art_v)]\n[media edit_video done: Cut to 9:16 -> video clip-vertical.mp4 (artifact art_out)]",
);
});
});

67
packages/core/src/attachments.ts
View File

@ -1,7 +1,9 @@
import {
ATTACHMENT_MAX_BASE64_LENGTH,
ATTACHMENT_MAX_BYTES,
type AttachmentKind,
type AttachmentMimeType,
attachmentKindOf,
isAllowedAttachmentMimeType,
isAttachmentImageMimeType,
type MessageBlock,
@ -45,8 +47,8 @@ export function validateAttachmentMimeType(mimeType: string): void {
}
}
export function attachmentKindForMimeType(mimeType: string): "image" | "file" {
return isAttachmentImageMimeType(mimeType) ? "image" : "file";
export function attachmentKindForMimeType(mimeType: string): AttachmentKind {
return attachmentKindOf(mimeType);
}
export function messageBlockForArtifact(artifact: {
@ -54,8 +56,12 @@ export function messageBlockForArtifact(artifact: {
name: string;
mimeType: string;
size: number;
}): Extract<MessageBlock, { kind: "image" | "file" }> {
if (attachmentKindForMimeType(artifact.mimeType) === "image") {
width?: number | null;
height?: number | null;
durationMs?: number | null;
}): Extract<MessageBlock, { kind: "image" | "video" | "audio" | "file" }> {
const kind = attachmentKindForMimeType(artifact.mimeType);
if (kind === "image") {
return {
kind: "image",
artifactId: artifact.id,
@ -63,6 +69,28 @@ export function messageBlockForArtifact(artifact: {
name: artifact.name,
};
}
if (kind === "video") {
return {
kind: "video",
artifactId: artifact.id,
mimeType: artifact.mimeType,
name: artifact.name,
size: artifact.size,
...(artifact.durationMs != null ? { durationMs: artifact.durationMs } : {}),
...(artifact.width != null ? { width: artifact.width } : {}),
...(artifact.height != null ? { height: artifact.height } : {}),
};
}
if (kind === "audio") {
return {
kind: "audio",
artifactId: artifact.id,
mimeType: artifact.mimeType,
name: artifact.name,
size: artifact.size,
...(artifact.durationMs != null ? { durationMs: artifact.durationMs } : {}),
};
}
return {
kind: "file",
artifactId: artifact.id,
@ -95,6 +123,16 @@ export function blocksToAgentHistoryText(blocks: MessageBlock[]): string {
if (block.kind === "file") {
return `[file: ${block.name} (${block.mimeType}, ${block.size} bytes)]`;
}
// Media keeps the artifact id so a later turn can edit or export what it refers to.
if (block.kind === "video" || block.kind === "audio") {
return `[${block.kind}: ${block.name} (artifact ${block.artifactId})]`;
}
if (block.kind === "media") {
const outputs = block.outputs
.map((output) => `${output.kind} ${output.name} (artifact ${output.artifactId})`)
.join(", ");
return `[media ${block.operation} ${block.status}: ${block.title}${outputs ? ` -> ${outputs}` : ""}]`;
}
// Keep attribution on peer messages: without it a later turn cannot tell
// which lines came from another bot rather than the user.
if (block.kind === "bot_message_received") return `[from ${block.fromBotName}] ${block.text}`;
@ -124,6 +162,17 @@ const EXTENSION_MIME_TYPES: Record<string, AttachmentMimeType> = {
".markdown": "text/markdown",
".csv": "text/csv",
".json": "application/json",
".mp4": "video/mp4",
".m4v": "video/mp4",
".webm": "video/webm",
".mov": "video/quicktime",
".mp3": "audio/mpeg",
".m4a": "audio/mp4",
".wav": "audio/wav",
".ogg": "audio/ogg",
".oga": "audio/ogg",
".vtt": "text/vtt",
".srt": "application/x-subrip",
};
const MIME_TYPE_EXTENSIONS: Record<AttachmentMimeType, string> = {
@ -136,6 +185,16 @@ const MIME_TYPE_EXTENSIONS: Record<AttachmentMimeType, string> = {
"text/markdown": ".md",
"text/csv": ".csv",
"application/json": ".json",
"video/mp4": ".mp4",
"video/webm": ".webm",
"video/quicktime": ".mov",
"audio/mpeg": ".mp3",
"audio/mp4": ".m4a",
"audio/wav": ".wav",
"audio/webm": ".weba",
"audio/ogg": ".ogg",
"text/vtt": ".vtt",
"application/x-subrip": ".srt",
};
export function inferAttachmentMimeType(

57
packages/db/prisma/migrations/20261003130000_media_artifacts_and_jobs/migration.sql Normal file
View File

@ -0,0 +1,57 @@
-- Media: artifacts remember their pixel size, play length and the artifact they were made from,
-- and generations, edits and transcriptions run as media jobs outside the agent turn.
ALTER TABLE "artifacts" ADD COLUMN "width" INTEGER;
ALTER TABLE "artifacts" ADD COLUMN "height" INTEGER;
ALTER TABLE "artifacts" ADD COLUMN "durationMs" INTEGER;
ALTER TABLE "artifacts" ADD COLUMN "parentArtifactId" TEXT;
CREATE INDEX "artifacts_parentArtifactId_idx" ON "artifacts"("parentArtifactId");
ALTER TABLE "artifacts" ADD CONSTRAINT "artifacts_parentArtifactId_fkey"
FOREIGN KEY ("parentArtifactId") REFERENCES "artifacts"("id") ON DELETE SET NULL ON UPDATE CASCADE
NOT VALID;
ALTER TABLE "artifacts" ADD CONSTRAINT "artifacts_media_metadata_chk"
CHECK (
("width" IS NULL OR "width" > 0)
AND ("height" IS NULL OR "height" > 0)
AND ("durationMs" IS NULL OR "durationMs" >= 0)
) NOT VALID;
CREATE TABLE "media_jobs" (
"id" TEXT NOT NULL,
"spaceId" TEXT NOT NULL,
"userId" TEXT NOT NULL,
"botId" TEXT,
"groupId" TEXT,
"threadId" TEXT NOT NULL,
"messageId" TEXT,
"runId" TEXT,
"operation" TEXT NOT NULL,
"status" TEXT NOT NULL DEFAULT 'queued',
"title" TEXT NOT NULL DEFAULT '',
"input" JSONB NOT NULL,
"progress" DOUBLE PRECISION,
"error" TEXT,
"outputArtifactIds" TEXT[] DEFAULT ARRAY[]::TEXT[],
"attempts" INTEGER NOT NULL DEFAULT 0,
"cancelRequested" BOOLEAN NOT NULL DEFAULT false,
"startedAt" TIMESTAMP(3),
"finishedAt" TIMESTAMP(3),
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "media_jobs_pkey" PRIMARY KEY ("id"),
CONSTRAINT "media_jobs_operation_chk" CHECK ("operation" IN (
'generate_image', 'generate_video', 'edit_image', 'edit_video', 'export_video', 'transcribe'
)),
CONSTRAINT "media_jobs_status_chk" CHECK ("status" IN (
'queued', 'waiting', 'running', 'done', 'failed', 'cancelled'
)),
CONSTRAINT "media_jobs_progress_chk" CHECK ("progress" IS NULL OR ("progress" >= 0 AND "progress" <= 1))
);
CREATE INDEX "media_jobs_spaceId_status_idx" ON "media_jobs"("spaceId", "status");
CREATE INDEX "media_jobs_threadId_idx" ON "media_jobs"("threadId");
ALTER TABLE "media_jobs" ADD CONSTRAINT "media_jobs_spaceId_fkey"
FOREIGN KEY ("spaceId") REFERENCES "spaces"("id") ON DELETE CASCADE ON UPDATE CASCADE;

4
packages/db/prisma/migrations/20261003130001_validate_media_artifacts_and_jobs_constraints/migration.sql Normal file
View File

@ -0,0 +1,4 @@
-- Kept separate from creation so PostgreSQL validates existing rows with a SHARE UPDATE
-- EXCLUSIVE lock (reads and writes continue) after the NOT VALID constraints have committed.
ALTER TABLE "artifacts" VALIDATE CONSTRAINT "artifacts_parentArtifactId_fkey";
ALTER TABLE "artifacts" VALIDATE CONSTRAINT "artifacts_media_metadata_chk";

42
packages/db/prisma/schema.prisma
View File

@ -197,6 +197,7 @@ model Space {
agentHomes AgentHome[]
browserProfiles BrowserProfile[]
artifacts Artifact[]
mediaJobs MediaJob[]
usageRecords UsageRecord[]
notificationPreferences NotificationPreference[]
computers Computer[]
@ -1065,13 +1066,54 @@ model Artifact {
size Int
hash String
storageKey String
/// Pixel size of images and videos and play length of audio and video, when known.
width Int?
height Int?
durationMs Int?
/// The artifact this one was made from (an edit, cut or export), for version history.
parentArtifactId String?
parent Artifact? @relation("ArtifactVersions", fields: [parentArtifactId], references: [id], onDelete: SetNull)
versions Artifact[] @relation("ArtifactVersions")
createdAt DateTime @default(now())
@@index([spaceId, botId])
@@index([groupId])
@@index([parentArtifactId])
@@map("artifacts")
}
/// A generation, edit or transcription that runs outside the agent turn and reports into a
/// media card in the conversation. Like CloudAgent, provenance ids carry no foreign keys.
model MediaJob {
id String @id @default(cuid())
spaceId String
space Space @relation(fields: [spaceId], references: [id], onDelete: Cascade)
userId String
botId String?
groupId String?
threadId String
/// The bot message that carries the media card.
messageId String?
runId String?
operation String
status String @default("queued")
title String @default("")
input Json
progress Float?
error String?
outputArtifactIds String[] @default([])
attempts Int @default(0)
cancelRequested Boolean @default(false)
startedAt DateTime?
finishedAt DateTime?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@index([spaceId, status])
@@index([threadId])
@@map("media_jobs")
}
model UsageRecord {
id String @id @default(cuid())
spaceId String