From e8f8912c658185655a970445acbdd5a130c69574 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 19:21:45 +0000 Subject: [PATCH] feat(media): wire agent tools and skills, ffmpeg in compose image Hook the media_* tools and the Video Producer and Video Copy skills into the executor, pi-runtime activity labels and the skill catalog; all stay hidden unless Media Studio is configured. Install ffmpeg in the compose image for renders and drop a local path from ARCHITECTURE.md. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w --- ARCHITECTURE.md | 4 +-- apps/api/src/agent-skills.ts | 18 ++++++---- apps/api/src/app.ts | 2 ++ apps/api/src/router.ts | 4 ++- apps/worker/src/index.ts | 2 ++ infra/compose/Dockerfile | 1 + packages/adapters/src/builtin-skills.ts | 2 ++ packages/adapters/src/builtin-tools.ts | 3 ++ packages/adapters/src/executor-types.ts | 3 ++ packages/adapters/src/executor.ts | 46 ++++++++++++++++++------- packages/adapters/src/index.ts | 1 + packages/adapters/src/pi-runtime.ts | 3 ++ 12 files changed, 67 insertions(+), 22 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 787c46c..1d28475 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -3,7 +3,7 @@ > **Status**: RATIFIZIERT & AKTIV > **Ecosystem**: evOo Autonomous Agent Platform > **Executive Lead / CTO**: Steve Jobs (`templates/cto-steve-jobs`) -> **Konfigurationsanker**: [`evoo.config.json`](file:///Users/ingowagner/Projects/evOo-local/evoo.config.json) +> **Konfigurationsanker**: [`evoo.config.json`](evoo.config.json) > **Grundsatz**: *Realität im Dateisystem vor Halluzination im Kopf. Nur was physisch geschrieben, getestet und lauffähig ist, existiert.* --- @@ -62,7 +62,7 @@ Das evOo-Ökosystem ist in klar getrennte, orthogonale Säulen unterteilt: ``` -/Users/ingowagner/Projects/evOo-local/ +evOo-local/ ├── ARCHITECTURE.md # Diese Verfassung ├── evoo.config.json # Autonome Berechtigungen & Policies ├── deploy.sh # Hot-Reload, Health-Checks & Service-Orchestrierung diff --git a/apps/api/src/agent-skills.ts b/apps/api/src/agent-skills.ts index a701ffd..5559675 100644 --- a/apps/api/src/agent-skills.ts +++ b/apps/api/src/agent-skills.ts @@ -1,5 +1,5 @@ import { ORPCError } from "@orpc/server"; -import { BUILTIN_AGENT_SKILLS } from "@rakazo/adapters"; +import { BUILTIN_AGENT_SKILLS, filterMediaSkills } from "@rakazo/adapters"; import type { Actor, AgentSkill, AgentSkillSource } from "@rakazo/contracts"; import { buildSkillMd, @@ -118,7 +118,13 @@ export function resolveSkillContent(input: { }; } -export function createAgentSkillsService(prisma: PrismaClient) { +export function createAgentSkillsService( + prisma: PrismaClient, + options: { mediaConfigured?: boolean } = {}, +) { + /** Built-in skills this deployment can run: media recipes only with Media Studio. */ + const visibleBuiltins = () => + filterMediaSkills(builtinCatalog(), Boolean(options.mediaConfigured)); async function owned(actor: Actor, skillId: string) { const row = await prisma.agentSkill.findFirst({ where: { @@ -137,7 +143,7 @@ export function createAgentSkillsService(prisma: PrismaClient) { where: { spaceId: actor.spaceId, userId: actor.userId }, orderBy: [{ name: "asc" }, { id: "asc" }], }); - return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill)).map( + return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill)).map( ({ content: _content, ...entry }) => entry, ); }, @@ -147,12 +153,12 @@ export function createAgentSkillsService(prisma: PrismaClient) { where: { spaceId: actor.spaceId, userId: actor.userId }, orderBy: [{ name: "asc" }, { id: "asc" }], }); - return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill)); + return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill)); }, async get(actor: Actor, input: { skillId?: string; name?: string }): Promise { if (input.skillId?.startsWith("builtin:")) { - const builtin = builtinCatalog().find((skill) => skill.id === input.skillId); + const builtin = visibleBuiltins().find((skill) => skill.id === input.skillId); if (!builtin) throw new IsolationError(); return builtin; } @@ -169,7 +175,7 @@ export function createAgentSkillsService(prisma: PrismaClient) { }); const row = findSkillByName(rows, name); if (row) return mapAgentSkill(row); - const builtin = findSkillByName(builtinCatalog(), name); + const builtin = findSkillByName(visibleBuiltins(), name); if (!builtin) throw new IsolationError(); return builtin; }, diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 131eef4..ee5cda5 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -415,6 +415,8 @@ export async function createApp( web: createWebProvider(), cloudAgent, shutdownSignal: shutdown.signal, + // Late-bound: media services are built below and need executor.resolveModel. + media: () => mediaServices, }); const mediaServices = createMediaServices({ diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index b032817..a866ef1 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -522,7 +522,9 @@ export function createRouter(deps: RouterDeps) { home: deps.home, dataDir: deps.dataDir, }); - const agentSkills = createAgentSkillsService(deps.prisma); + const agentSkills = createAgentSkillsService(deps.prisma, { + mediaConfigured: Boolean(deps.mediaServices), + }); const authed = os.use(async ({ context, next }) => { if (!context.actor) throw new ORPCError("UNAUTHORIZED"); diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index e0ef77a..fa194a6 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -202,6 +202,8 @@ async function main() { messaging: messaging ? createMessagingContextLoader(prisma) : undefined, web: createWebProvider(), cloudAgent, + // Late-bound: media services are built below and need executor.resolveModel. + media: () => mediaServices, }); const mediaAbort = new AbortController(); diff --git a/infra/compose/Dockerfile b/infra/compose/Dockerfile index fa6b508..d55e178 100644 --- a/infra/compose/Dockerfile +++ b/infra/compose/Dockerfile @@ -1,5 +1,6 @@ FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a RUN corepack enable +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* WORKDIR /app # Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise # leave the copied tree unreadable to USER node at runtime. diff --git a/packages/adapters/src/builtin-skills.ts b/packages/adapters/src/builtin-skills.ts index 063f91b..5b2b338 100644 --- a/packages/adapters/src/builtin-skills.ts +++ b/packages/adapters/src/builtin-skills.ts @@ -1,4 +1,5 @@ import { buildSkillMd } from "@rakazo/core"; +import { MEDIA_AGENT_SKILLS } from "./media-skills.js"; /** * Built-in Agent Skills (SKILL.md recipes) available to every user. @@ -21,6 +22,7 @@ const SKILLS: Array<{ name: string; description: string; body: string }> = [ 3. Verify before accusing: for each suspected issue, re-read the code and construct the concrete input or state that triggers the failure. Drop anything you cannot substantiate. 4. Synthesize a verdict: ship, ship after fixes, or do not ship. List the confirmed findings ordered by severity, each with its location and failure scenario, then any open questions. If nothing survived verification, say so plainly instead of inventing nitpicks.`, }, + ...MEDIA_AGENT_SKILLS, ]; export const BUILTIN_AGENT_SKILLS: Array<{ diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 0ff52fa..9cc5cbb 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -8,6 +8,7 @@ import { USER_FACT_CATEGORIES, } from "@rakazo/contracts"; import { z } from "zod"; +import { mediaAgentTools } from "./media-tools.js"; export const SUBAGENT_MODEL_TIER_DESCRIPTION = "Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set."; @@ -963,6 +964,8 @@ export const builtinAgentTools: ConnectorTool[] = [ required: ["bot", "title", "brief"], }, }, + // Offered only when Media Studio is configured (filterMediaTools in the executor). + ...mediaAgentTools, ]; /** Agent-connection tools, exposed only when the messaging surface is enabled. */ diff --git a/packages/adapters/src/executor-types.ts b/packages/adapters/src/executor-types.ts index c32ea62..82aca5f 100644 --- a/packages/adapters/src/executor-types.ts +++ b/packages/adapters/src/executor-types.ts @@ -13,6 +13,7 @@ import type { } from "@rakazo/adapter-kit"; import type { PrismaClient, ThreadEvents } from "@rakazo/db"; import type { CloudAgentConnection } from "./cloud-agent-factory.js"; +import type { MediaServices } from "./media-context.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js"; import type { RemoteTransportDependencies } from "./remote-mcp.js"; import type { EncryptedSecretStore } from "./secrets.js"; @@ -45,6 +46,8 @@ export interface ExecutorDeps { secretHttp?: RemoteTransportDependencies; /** Remote cloud coding agents. Null/omit means tools stay uninjected. */ cloudAgent?: CloudAgentConnection | null; + /** Media Studio services, read per run; null or omitted keeps the media_* tools hidden. */ + media?: () => MediaServices | null; /** Aborted when createApp stop() begins so in-flight continueRun boot waits exit promptly. */ shutdownSignal?: AbortSignal; } diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts index 199c914..4195cc9 100644 --- a/packages/adapters/src/executor.ts +++ b/packages/adapters/src/executor.ts @@ -290,6 +290,9 @@ import { needsOAuthProbe, parseMcpServerToolArgs, } from "./mcp-server-tool.js"; +import { filterMediaSkills } from "./media-skills.js"; +import { filterMediaTools, isMediaToolName } from "./media-tools.js"; +import { executeMediaTool } from "./media-tools-execute.js"; import { loadAgentMemoryContext } from "./memory-context.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js"; import { selectMemoryTools } from "./memory-tools.js"; @@ -1127,7 +1130,7 @@ export function createRunExecutor(deps: ExecutorDeps) { listAgentSkillRecords(deps.prisma, { spaceId: run.spaceId, userId: run.userId, - }), + }).then((records) => filterMediaSkills(records, Boolean(deps.media?.()))), deps.prisma.agentSecret.findMany({ where: agentSecretScopeForRun(run.spaceId, run.userId), select: { @@ -1477,18 +1480,22 @@ export function createRunExecutor(deps: ExecutorDeps) { select: { id: true }, })) : false; - const builtins = selectBuiltinToolsForRun({ - graphicalToolsAllowed, - pageBrowserAllowed, - groupId: thread.groupId, - trigger: run.trigger, - semanticMemoryEnabled, - cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId), - messagingChannelRun, - agentConnectionsEnabled: hasMessagingIdentity, - foreignAgentRun, - bot, - }); + const mediaServices = deps.media?.() ?? null; + const builtins = filterMediaTools( + selectBuiltinToolsForRun({ + graphicalToolsAllowed, + pageBrowserAllowed, + groupId: thread.groupId, + trigger: run.trigger, + semanticMemoryEnabled, + cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId), + messagingChannelRun, + agentConnectionsEnabled: hasMessagingIdentity, + foreignAgentRun, + bot, + }), + mediaServices !== null, + ); const exposedConnectorTools = discovered.filter( (tool) => !builtinAgentTools.some((builtin) => builtin.name === tool.name), ); @@ -2635,6 +2642,16 @@ export function createRunExecutor(deps: ExecutorDeps) { ); } + if (isMediaToolName(name)) { + return finish( + await executeMediaTool( + mediaServices, + { spaceId: run.spaceId, userId: run.userId }, + name, + args, + ), + ); + } if (name.startsWith("cloud_agent_")) { return finish( await executeCloudAgentTool( @@ -3824,6 +3841,9 @@ export function createRunExecutor(deps: ExecutorDeps) { pluginLine, agentSkillsLine, taughtSkillsLine, + exposedToolNames.has("media_search") + ? "For photo and video work, read the Video Producer skill with skill_read before using the media_* tools." + : undefined, exposedToolNames.has("render_plot") ? 'For charts and data visualization, use the render_plot tool: it renders bar, line, scatter, histogram, heatmap, faceted and many more chart types from a JSON spec and attaches the PNG to the chat. Call render_plot with {"help": true} before your first chart to read the full guide.' : undefined, diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 6f7dd24..f1fc684 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -118,6 +118,7 @@ export { } from "./media-render.js"; export * from "./media-services.js"; export * from "./media-sidecar.js"; +export * from "./media-skills.js"; export * from "./media-zip.js"; export * from "./memory-provider-factory.js"; export * from "./messaging-context.js"; diff --git a/packages/adapters/src/pi-runtime.ts b/packages/adapters/src/pi-runtime.ts index 95f295c..0153799 100644 --- a/packages/adapters/src/pi-runtime.ts +++ b/packages/adapters/src/pi-runtime.ts @@ -18,6 +18,7 @@ import { MODEL_TIERS, ModelTierSchema } from "@rakazo/contracts"; import { getLogger } from "@rakazo/logging"; import { isToolPauseResult } from "./approval-effect.js"; import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js"; +import { describeMediaToolActivity } from "./media-tools.js"; import { normalizeOpenAiToolParameters, openAiToolParametersNeedNormalization, @@ -474,6 +475,8 @@ export function describeToolActivity(toolName: string, args: unknown): string { if (toolName === "attach_file") return `Attaching ${detail(record.path)}`; if (toolName === "open_path") return `Opening ${detail(record.path)}`; if (toolName === "render_plot") return "Rendering a chart"; + const media = describeMediaToolActivity(toolName, record); + if (media) return media; if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`; if (toolName === "computer_observe") return "Looking at the screen"; if (toolName === "browser_navigate")