Groundwork for image and video generation, video editing and transcription. Nothing calls it yet; the media runner, providers, tools and UI build on it. - Contracts: video, audio and subtitle MIME types, per-kind size caps for streamed and generated artifacts, video/audio message blocks and the live media card block, media operations, job statuses and output presets (9:16, 16:9, 1:1, 4:5, 4:3, 21:9). - Core: attachment kinds include video and audio, artifacts become video or audio blocks, and agent history keeps media artifact ids so later turns can refer to them. - Schema: artifacts record width, height, duration and the artifact they were made from; media_jobs tracks generations, edits and transcriptions. - Adapter kit: provider-neutral image generation, video generation and transcription interfaces, the media job context operations run in, and the MediaRuntime that tools use to start jobs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
345 lines
10 KiB
TypeScript
345 lines
10 KiB
TypeScript
import * as z from "zod";
|
|
import { BotSecretDestination } from "./bot-secrets.js";
|
|
import { Id } from "./ids.js";
|
|
import { McpTransportSchema } from "./mcp.js";
|
|
import { AudioBlockSchema, MediaJobBlockSchema, VideoBlockSchema } from "./media.js";
|
|
|
|
export const ProductEventType = z.enum([
|
|
"thread.message.created",
|
|
"thread.cleared",
|
|
"thread.message.updated",
|
|
"thread.message.reaction",
|
|
"thread.progress",
|
|
"thread.artifact",
|
|
"thread.ask",
|
|
"thread.choice",
|
|
"thread.meta",
|
|
"thread.computer",
|
|
"thread.subagent",
|
|
"thread.cloud_agent",
|
|
"run.started",
|
|
"run.checkpointed",
|
|
"run.waiting_input",
|
|
"run.completed",
|
|
"run.failed",
|
|
"run.cancelled",
|
|
"computer.status",
|
|
"computer.takeover.requested",
|
|
"computer.takeover.granted",
|
|
"computer.takeover.released",
|
|
"memory.revised",
|
|
"routine.created",
|
|
"routine.updated",
|
|
"routine.fired",
|
|
"skill.teaching.started",
|
|
"skill.teaching.stopped",
|
|
"skill.draft.created",
|
|
"skill.saved",
|
|
"effect.recorded",
|
|
"agent.tool.called",
|
|
"agent.tool.completed",
|
|
"effect.reconciled",
|
|
"usage.recorded",
|
|
"bot.spawned",
|
|
"bot.updated",
|
|
"bot.archived",
|
|
"bot.deleted",
|
|
"group.created",
|
|
"group.updated",
|
|
"group.handoff",
|
|
]);
|
|
export type ProductEventType = z.infer<typeof ProductEventType>;
|
|
|
|
export const MessageRole = z.enum(["user", "bot", "system"]);
|
|
export const BotMessageIntent = z.enum(["request", "result", "question", "status", "fyi"]);
|
|
export type BotMessageIntent = z.infer<typeof BotMessageIntent>;
|
|
|
|
export const MAX_CHART_DATA_ROWS = 5_000;
|
|
|
|
const ChartSpec = z.record(z.string(), z.any());
|
|
|
|
function embeddedChartRowCount(spec: Record<string, unknown>): number {
|
|
const specData = Array.isArray(spec.data) ? spec.data.length : 0;
|
|
const markData = Array.isArray(spec.marks)
|
|
? spec.marks.reduce((total, mark) => {
|
|
if (!mark || typeof mark !== "object" || !Array.isArray(mark.data)) return total;
|
|
return total + mark.data.length;
|
|
}, 0)
|
|
: 0;
|
|
return specData + markData;
|
|
}
|
|
|
|
const ChartBlock = z
|
|
.object({
|
|
kind: z.literal("chart"),
|
|
name: z.string(),
|
|
/** Declarative Observable Plot spec, validated by render_plot before publish.
|
|
z.any keeps the inferred type JSON-assignable for persistence. */
|
|
spec: ChartSpec,
|
|
data: z.array(z.any()).max(MAX_CHART_DATA_ROWS),
|
|
})
|
|
.superRefine((block, ctx) => {
|
|
if (block.data.length + embeddedChartRowCount(block.spec) <= MAX_CHART_DATA_ROWS) return;
|
|
ctx.addIssue({
|
|
code: "custom",
|
|
message: `Chart data exceeds the ${MAX_CHART_DATA_ROWS.toLocaleString("en-US")}-row limit`,
|
|
});
|
|
});
|
|
|
|
export const SecretAskPurpose = z.enum(["otp", "password", "api_key"]);
|
|
export type SecretAskPurpose = z.infer<typeof SecretAskPurpose>;
|
|
|
|
/** Lives here rather than in domain.ts because the task card block carries it. */
|
|
export const DELEGATED_TASK_STATUSES = [
|
|
"queued",
|
|
"running",
|
|
"waiting",
|
|
"done",
|
|
"failed",
|
|
"cancelled",
|
|
] as const;
|
|
export const DelegatedTaskStatusSchema = z.enum(DELEGATED_TASK_STATUSES);
|
|
export type DelegatedTaskStatus = z.infer<typeof DelegatedTaskStatusSchema>;
|
|
|
|
/** A question or approval that pauses its run until the user answers. */
|
|
export const AskBlockSchema = z.object({
|
|
kind: z.literal("ask"),
|
|
text: z.string(),
|
|
approvalEffectId: Id.optional(),
|
|
detail: z.string().optional(),
|
|
input: z.enum(["text", "secret"]).optional(),
|
|
/** Why the secret is needed; drives field label on the masked card. */
|
|
purpose: SecretAskPurpose.optional(),
|
|
credential: BotSecretDestination.optional(),
|
|
status: z.enum(["pending", "answered"]).optional(),
|
|
answer: z.string().optional(),
|
|
actions: z
|
|
.array(
|
|
z.object({
|
|
id: z.string(),
|
|
label: z.string(),
|
|
outcome: z.enum(["created", "cancelled"]).optional(),
|
|
}),
|
|
)
|
|
.optional(),
|
|
});
|
|
|
|
export const MessageBlock = z.discriminatedUnion("kind", [
|
|
z.object({ kind: z.literal("text"), text: z.string() }),
|
|
z.object({
|
|
kind: z.literal("card"),
|
|
lines: z.array(z.object({ k: z.string(), v: z.string() })),
|
|
}),
|
|
AskBlockSchema,
|
|
z.object({
|
|
kind: z.literal("choice"),
|
|
question: z.string(),
|
|
subtitle: z.string().optional(),
|
|
options: z.array(z.object({ id: z.string(), letter: z.string(), label: z.string() })),
|
|
/** Set once the user picks an option; renders the picker as answered. */
|
|
answerId: z.string().optional(),
|
|
}),
|
|
z.object({
|
|
/** Inline app authorization card (Composio-backed): logo, name, one-line
|
|
description, and an Authorize button that flips to connected. */
|
|
kind: z.literal("app_connect"),
|
|
connectorId: z.string().optional(),
|
|
provider: z.string(),
|
|
name: z.string(),
|
|
description: z.string(),
|
|
logo: z.string().nullable(),
|
|
status: z.enum(["pending", "connected"]),
|
|
}),
|
|
z.object({
|
|
kind: z.literal("connect"),
|
|
name: z.string(),
|
|
initial: z.string(),
|
|
color: z.string(),
|
|
status: z.enum(["pending", "connected"]),
|
|
}),
|
|
z.object({
|
|
kind: z.literal("computer"),
|
|
state: z.string(),
|
|
text: z.string(),
|
|
}),
|
|
z.object({ kind: z.literal("meta"), text: z.string() }),
|
|
z.object({
|
|
kind: z.literal("progress"),
|
|
text: z.string(),
|
|
/** Provider-generated tool status rather than assistant-authored narration. */
|
|
activity: z.literal(true).optional(),
|
|
pendingToolNames: z.array(z.string()).optional(),
|
|
}),
|
|
z.object({
|
|
kind: z.literal("steps"),
|
|
steps: z.array(z.object({ label: z.string(), count: z.number().int().positive() })),
|
|
durationMs: z.number().int().nonnegative().optional(),
|
|
}),
|
|
z.object({
|
|
kind: z.literal("subagent"),
|
|
agentId: z.string(),
|
|
name: z.string(),
|
|
task: z.string(),
|
|
status: z.enum(["running", "completed", "failed"]),
|
|
progress: z.string().optional(),
|
|
result: z.string().optional(),
|
|
}),
|
|
z.object({
|
|
kind: z.literal("child_bot"),
|
|
botId: z.string(),
|
|
name: z.string(),
|
|
title: z.string().optional(),
|
|
status: z.enum(["created", "archived", "deleted"]),
|
|
}),
|
|
z.object({
|
|
/** Clickable card when an agent creates a project from chat. */
|
|
kind: z.literal("project_created"),
|
|
groupId: Id,
|
|
name: z.string(),
|
|
memberCount: z.number().int().positive(),
|
|
}),
|
|
z.object({
|
|
/** Compact card for a remote cloud coding agent (not the bot computer). */
|
|
kind: z.literal("cloud_agent"),
|
|
agentId: z.string(),
|
|
title: z.string(),
|
|
status: z.enum(["running", "finished", "failed", "cancelled"]),
|
|
url: z.string(),
|
|
branch: z.string().optional(),
|
|
prUrl: z.string().optional(),
|
|
latestRunId: z.string().optional(),
|
|
}),
|
|
z.object({
|
|
kind: z.literal("skill_draft"),
|
|
skillId: Id,
|
|
name: z.string(),
|
|
goal: z.string(),
|
|
playbook: z.object({
|
|
whenToUse: z.string(),
|
|
inputs: z.array(z.string()),
|
|
steps: z.array(z.string()),
|
|
howToCheck: z.string(),
|
|
whatToReturn: z.string(),
|
|
approvalBoundaries: z.string(),
|
|
failureHandling: z.string(),
|
|
}),
|
|
status: z.enum(["draft", "saved"]),
|
|
}),
|
|
ChartBlock,
|
|
z.object({
|
|
/** Approval card for an agent-created MCP server. The user completes the
|
|
OAuth popup (or confirms no authorization is needed) in the UI. */
|
|
kind: z.literal("mcp_approval"),
|
|
name: z.string(),
|
|
serverId: Id,
|
|
transport: McpTransportSchema,
|
|
endpoint: z.string().nullable(),
|
|
needsOAuth: z.boolean(),
|
|
}),
|
|
z.object({
|
|
kind: z.literal("image"),
|
|
artifactId: Id,
|
|
mimeType: z.string(),
|
|
name: z.string(),
|
|
}),
|
|
z.object({
|
|
kind: z.literal("file"),
|
|
artifactId: Id,
|
|
mimeType: z.string(),
|
|
name: z.string(),
|
|
size: z.number().int().nonnegative(),
|
|
}),
|
|
VideoBlockSchema,
|
|
AudioBlockSchema,
|
|
MediaJobBlockSchema,
|
|
z.object({
|
|
kind: z.literal("handoff"),
|
|
fromBotId: Id,
|
|
toBotId: Id,
|
|
text: z.string(),
|
|
/** Links ownership transfers in one user-started group turn. */
|
|
hop: z.number().int().positive().optional(),
|
|
}),
|
|
z.object({
|
|
/** A group-chat message delivered into a member bot's own thread. */
|
|
kind: z.literal("channel_message"),
|
|
provider: z.string(),
|
|
/** Per-message network when a provider spans multiple transports. */
|
|
transport: z.string().optional(),
|
|
channelId: Id,
|
|
fromAddress: z.string(),
|
|
fromLabel: z.string(),
|
|
text: z.string(),
|
|
hop: z.number().int().nonnegative().optional(),
|
|
}),
|
|
z.object({
|
|
/** Shown in the sending bot's own chat, so the user can see what it sent. */
|
|
kind: z.literal("bot_message_sent"),
|
|
toBotId: Id,
|
|
toBotName: z.string(),
|
|
text: z.string(),
|
|
intent: BotMessageIntent.optional(),
|
|
}),
|
|
z.object({
|
|
/** Work the main bot handed to a specialist; status follows the task live. */
|
|
kind: z.literal("task"),
|
|
taskId: Id,
|
|
assigneeBotId: Id,
|
|
assigneeBotName: z.string(),
|
|
title: z.string(),
|
|
/** The brief the specialist works from. */
|
|
text: z.string(),
|
|
status: DelegatedTaskStatusSchema,
|
|
/** Links delegations started from a task result; absent when a person started it. */
|
|
hop: z.number().int().positive().optional(),
|
|
}),
|
|
z.object({
|
|
/** Delivered into the receiving bot's own chat as the prompt that woke it. */
|
|
kind: z.literal("bot_message_received"),
|
|
fromBotId: Id,
|
|
fromBotName: z.string(),
|
|
text: z.string(),
|
|
intent: BotMessageIntent.optional(),
|
|
/** Sender-thread echo this delivery answers, when applicable. */
|
|
returnToMessageId: Id.optional(),
|
|
/** Links in a bot-started chain; absent when a person started it. */
|
|
hop: z.number().int().nonnegative().optional(),
|
|
}),
|
|
]);
|
|
export type MessageBlock = z.infer<typeof MessageBlock>;
|
|
|
|
export const ProductEventSchema = z.object({
|
|
id: Id,
|
|
spaceId: Id,
|
|
threadId: Id,
|
|
botId: Id,
|
|
seq: z.number().int().nonnegative(),
|
|
type: ProductEventType,
|
|
runId: Id.optional(),
|
|
createdAt: z.string(),
|
|
payload: z.record(z.string(), z.unknown()),
|
|
});
|
|
export type ProductEvent = z.infer<typeof ProductEventSchema>;
|
|
|
|
export const ThreadMessageSchema = z.object({
|
|
id: Id,
|
|
threadId: Id,
|
|
seq: z.number().int().nonnegative(),
|
|
role: MessageRole,
|
|
blocks: z.array(MessageBlock),
|
|
botId: Id.optional(),
|
|
replyToMessageId: Id.optional(),
|
|
replyQuote: z.string().optional(),
|
|
runId: Id.optional(),
|
|
createdAt: z.string(),
|
|
});
|
|
export type ThreadMessage = z.infer<typeof ThreadMessageSchema>;
|
|
|
|
export function canReactToThreadMessage(message: Pick<ThreadMessage, "id" | "blocks">): boolean {
|
|
return (
|
|
!message.id.startsWith("progress:") &&
|
|
!message.id.startsWith("subagent:") &&
|
|
!message.blocks.some((block) => block.kind === "channel_message")
|
|
);
|
|
}
|