feat(media): wire agent tools and skills, ffmpeg in compose image

Hook the media_* tools and the Video Producer and Video Copy skills into the
executor, pi-runtime activity labels and the skill catalog; all stay hidden
unless Media Studio is configured. Install ffmpeg in the compose image for
renders and drop a local path from ARCHITECTURE.md.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w
This commit is contained in:
Claude 2026-10-03 19:21:45 +00:00
parent 80f082f910
commit e8f8912c65
No known key found for this signature in database
12 changed files with 67 additions and 22 deletions

View File

@ -3,7 +3,7 @@
> **Status**: RATIFIZIERT & AKTIV > **Status**: RATIFIZIERT & AKTIV
> **Ecosystem**: evOo Autonomous Agent Platform > **Ecosystem**: evOo Autonomous Agent Platform
> **Executive Lead / CTO**: Steve Jobs (`templates/cto-steve-jobs`) > **Executive Lead / CTO**: Steve Jobs (`templates/cto-steve-jobs`)
> **Konfigurationsanker**: [`evoo.config.json`](file:///Users/ingowagner/Projects/evOo-local/evoo.config.json) > **Konfigurationsanker**: [`evoo.config.json`](evoo.config.json)
> **Grundsatz**: *Realität im Dateisystem vor Halluzination im Kopf. Nur was physisch geschrieben, getestet und lauffähig ist, existiert.* > **Grundsatz**: *Realität im Dateisystem vor Halluzination im Kopf. Nur was physisch geschrieben, getestet und lauffähig ist, existiert.*
--- ---
@ -62,7 +62,7 @@
Das evOo-Ökosystem ist in klar getrennte, orthogonale Säulen unterteilt: Das evOo-Ökosystem ist in klar getrennte, orthogonale Säulen unterteilt:
``` ```
/Users/ingowagner/Projects/evOo-local/ evOo-local/
├── ARCHITECTURE.md # Diese Verfassung ├── ARCHITECTURE.md # Diese Verfassung
├── evoo.config.json # Autonome Berechtigungen & Policies ├── evoo.config.json # Autonome Berechtigungen & Policies
├── deploy.sh # Hot-Reload, Health-Checks & Service-Orchestrierung ├── deploy.sh # Hot-Reload, Health-Checks & Service-Orchestrierung

View File

@ -1,5 +1,5 @@
import { ORPCError } from "@orpc/server"; import { ORPCError } from "@orpc/server";
import { BUILTIN_AGENT_SKILLS } from "@rakazo/adapters"; import { BUILTIN_AGENT_SKILLS, filterMediaSkills } from "@rakazo/adapters";
import type { Actor, AgentSkill, AgentSkillSource } from "@rakazo/contracts"; import type { Actor, AgentSkill, AgentSkillSource } from "@rakazo/contracts";
import { import {
buildSkillMd, buildSkillMd,
@ -118,7 +118,13 @@ export function resolveSkillContent(input: {
}; };
} }
export function createAgentSkillsService(prisma: PrismaClient) { export function createAgentSkillsService(
prisma: PrismaClient,
options: { mediaConfigured?: boolean } = {},
) {
/** Built-in skills this deployment can run: media recipes only with Media Studio. */
const visibleBuiltins = () =>
filterMediaSkills(builtinCatalog(), Boolean(options.mediaConfigured));
async function owned(actor: Actor, skillId: string) { async function owned(actor: Actor, skillId: string) {
const row = await prisma.agentSkill.findFirst({ const row = await prisma.agentSkill.findFirst({
where: { where: {
@ -137,7 +143,7 @@ export function createAgentSkillsService(prisma: PrismaClient) {
where: { spaceId: actor.spaceId, userId: actor.userId }, where: { spaceId: actor.spaceId, userId: actor.userId },
orderBy: [{ name: "asc" }, { id: "asc" }], orderBy: [{ name: "asc" }, { id: "asc" }],
}); });
return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill)).map( return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill)).map(
({ content: _content, ...entry }) => entry, ({ content: _content, ...entry }) => entry,
); );
}, },
@ -147,12 +153,12 @@ export function createAgentSkillsService(prisma: PrismaClient) {
where: { spaceId: actor.spaceId, userId: actor.userId }, where: { spaceId: actor.spaceId, userId: actor.userId },
orderBy: [{ name: "asc" }, { id: "asc" }], orderBy: [{ name: "asc" }, { id: "asc" }],
}); });
return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill)); return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill));
}, },
async get(actor: Actor, input: { skillId?: string; name?: string }): Promise<AgentSkill> { async get(actor: Actor, input: { skillId?: string; name?: string }): Promise<AgentSkill> {
if (input.skillId?.startsWith("builtin:")) { if (input.skillId?.startsWith("builtin:")) {
const builtin = builtinCatalog().find((skill) => skill.id === input.skillId); const builtin = visibleBuiltins().find((skill) => skill.id === input.skillId);
if (!builtin) throw new IsolationError(); if (!builtin) throw new IsolationError();
return builtin; return builtin;
} }
@ -169,7 +175,7 @@ export function createAgentSkillsService(prisma: PrismaClient) {
}); });
const row = findSkillByName(rows, name); const row = findSkillByName(rows, name);
if (row) return mapAgentSkill(row); if (row) return mapAgentSkill(row);
const builtin = findSkillByName(builtinCatalog(), name); const builtin = findSkillByName(visibleBuiltins(), name);
if (!builtin) throw new IsolationError(); if (!builtin) throw new IsolationError();
return builtin; return builtin;
}, },

View File

@ -415,6 +415,8 @@ export async function createApp(
web: createWebProvider(), web: createWebProvider(),
cloudAgent, cloudAgent,
shutdownSignal: shutdown.signal, shutdownSignal: shutdown.signal,
// Late-bound: media services are built below and need executor.resolveModel.
media: () => mediaServices,
}); });
const mediaServices = createMediaServices({ const mediaServices = createMediaServices({

View File

@ -522,7 +522,9 @@ export function createRouter(deps: RouterDeps) {
home: deps.home, home: deps.home,
dataDir: deps.dataDir, dataDir: deps.dataDir,
}); });
const agentSkills = createAgentSkillsService(deps.prisma); const agentSkills = createAgentSkillsService(deps.prisma, {
mediaConfigured: Boolean(deps.mediaServices),
});
const authed = os.use(async ({ context, next }) => { const authed = os.use(async ({ context, next }) => {
if (!context.actor) throw new ORPCError("UNAUTHORIZED"); if (!context.actor) throw new ORPCError("UNAUTHORIZED");

View File

@ -202,6 +202,8 @@ async function main() {
messaging: messaging ? createMessagingContextLoader(prisma) : undefined, messaging: messaging ? createMessagingContextLoader(prisma) : undefined,
web: createWebProvider(), web: createWebProvider(),
cloudAgent, cloudAgent,
// Late-bound: media services are built below and need executor.resolveModel.
media: () => mediaServices,
}); });
const mediaAbort = new AbortController(); const mediaAbort = new AbortController();

View File

@ -1,5 +1,6 @@
FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a
RUN corepack enable RUN corepack enable
RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
WORKDIR /app WORKDIR /app
# Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise # Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise
# leave the copied tree unreadable to USER node at runtime. # leave the copied tree unreadable to USER node at runtime.

View File

@ -1,4 +1,5 @@
import { buildSkillMd } from "@rakazo/core"; import { buildSkillMd } from "@rakazo/core";
import { MEDIA_AGENT_SKILLS } from "./media-skills.js";
/** /**
* Built-in Agent Skills (SKILL.md recipes) available to every user. * Built-in Agent Skills (SKILL.md recipes) available to every user.
@ -21,6 +22,7 @@ const SKILLS: Array<{ name: string; description: string; body: string }> = [
3. Verify before accusing: for each suspected issue, re-read the code and construct the concrete input or state that triggers the failure. Drop anything you cannot substantiate. 3. Verify before accusing: for each suspected issue, re-read the code and construct the concrete input or state that triggers the failure. Drop anything you cannot substantiate.
4. Synthesize a verdict: ship, ship after fixes, or do not ship. List the confirmed findings ordered by severity, each with its location and failure scenario, then any open questions. If nothing survived verification, say so plainly instead of inventing nitpicks.`, 4. Synthesize a verdict: ship, ship after fixes, or do not ship. List the confirmed findings ordered by severity, each with its location and failure scenario, then any open questions. If nothing survived verification, say so plainly instead of inventing nitpicks.`,
}, },
...MEDIA_AGENT_SKILLS,
]; ];
export const BUILTIN_AGENT_SKILLS: Array<{ export const BUILTIN_AGENT_SKILLS: Array<{

View File

@ -8,6 +8,7 @@ import {
USER_FACT_CATEGORIES, USER_FACT_CATEGORIES,
} from "@rakazo/contracts"; } from "@rakazo/contracts";
import { z } from "zod"; import { z } from "zod";
import { mediaAgentTools } from "./media-tools.js";
export const SUBAGENT_MODEL_TIER_DESCRIPTION = export const SUBAGENT_MODEL_TIER_DESCRIPTION =
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set."; "Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
@ -963,6 +964,8 @@ export const builtinAgentTools: ConnectorTool[] = [
required: ["bot", "title", "brief"], required: ["bot", "title", "brief"],
}, },
}, },
// Offered only when Media Studio is configured (filterMediaTools in the executor).
...mediaAgentTools,
]; ];
/** Agent-connection tools, exposed only when the messaging surface is enabled. */ /** Agent-connection tools, exposed only when the messaging surface is enabled. */

View File

@ -13,6 +13,7 @@ import type {
} from "@rakazo/adapter-kit"; } from "@rakazo/adapter-kit";
import type { PrismaClient, ThreadEvents } from "@rakazo/db"; import type { PrismaClient, ThreadEvents } from "@rakazo/db";
import type { CloudAgentConnection } from "./cloud-agent-factory.js"; import type { CloudAgentConnection } from "./cloud-agent-factory.js";
import type { MediaServices } from "./media-context.js";
import type { MemoryProviderResolver } from "./memory-provider-factory.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js";
import type { RemoteTransportDependencies } from "./remote-mcp.js"; import type { RemoteTransportDependencies } from "./remote-mcp.js";
import type { EncryptedSecretStore } from "./secrets.js"; import type { EncryptedSecretStore } from "./secrets.js";
@ -45,6 +46,8 @@ export interface ExecutorDeps {
secretHttp?: RemoteTransportDependencies; secretHttp?: RemoteTransportDependencies;
/** Remote cloud coding agents. Null/omit means tools stay uninjected. */ /** Remote cloud coding agents. Null/omit means tools stay uninjected. */
cloudAgent?: CloudAgentConnection | null; cloudAgent?: CloudAgentConnection | null;
/** Media Studio services, read per run; null or omitted keeps the media_* tools hidden. */
media?: () => MediaServices | null;
/** Aborted when createApp stop() begins so in-flight continueRun boot waits exit promptly. */ /** Aborted when createApp stop() begins so in-flight continueRun boot waits exit promptly. */
shutdownSignal?: AbortSignal; shutdownSignal?: AbortSignal;
} }

View File

@ -290,6 +290,9 @@ import {
needsOAuthProbe, needsOAuthProbe,
parseMcpServerToolArgs, parseMcpServerToolArgs,
} from "./mcp-server-tool.js"; } from "./mcp-server-tool.js";
import { filterMediaSkills } from "./media-skills.js";
import { filterMediaTools, isMediaToolName } from "./media-tools.js";
import { executeMediaTool } from "./media-tools-execute.js";
import { loadAgentMemoryContext } from "./memory-context.js"; import { loadAgentMemoryContext } from "./memory-context.js";
import type { MemoryProviderResolver } from "./memory-provider-factory.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js";
import { selectMemoryTools } from "./memory-tools.js"; import { selectMemoryTools } from "./memory-tools.js";
@ -1127,7 +1130,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
listAgentSkillRecords(deps.prisma, { listAgentSkillRecords(deps.prisma, {
spaceId: run.spaceId, spaceId: run.spaceId,
userId: run.userId, userId: run.userId,
}), }).then((records) => filterMediaSkills(records, Boolean(deps.media?.()))),
deps.prisma.agentSecret.findMany({ deps.prisma.agentSecret.findMany({
where: agentSecretScopeForRun(run.spaceId, run.userId), where: agentSecretScopeForRun(run.spaceId, run.userId),
select: { select: {
@ -1477,7 +1480,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
select: { id: true }, select: { id: true },
})) }))
: false; : false;
const builtins = selectBuiltinToolsForRun({ const mediaServices = deps.media?.() ?? null;
const builtins = filterMediaTools(
selectBuiltinToolsForRun({
graphicalToolsAllowed, graphicalToolsAllowed,
pageBrowserAllowed, pageBrowserAllowed,
groupId: thread.groupId, groupId: thread.groupId,
@ -1488,7 +1493,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
agentConnectionsEnabled: hasMessagingIdentity, agentConnectionsEnabled: hasMessagingIdentity,
foreignAgentRun, foreignAgentRun,
bot, bot,
}); }),
mediaServices !== null,
);
const exposedConnectorTools = discovered.filter( const exposedConnectorTools = discovered.filter(
(tool) => !builtinAgentTools.some((builtin) => builtin.name === tool.name), (tool) => !builtinAgentTools.some((builtin) => builtin.name === tool.name),
); );
@ -2635,6 +2642,16 @@ export function createRunExecutor(deps: ExecutorDeps) {
); );
} }
if (isMediaToolName(name)) {
return finish(
await executeMediaTool(
mediaServices,
{ spaceId: run.spaceId, userId: run.userId },
name,
args,
),
);
}
if (name.startsWith("cloud_agent_")) { if (name.startsWith("cloud_agent_")) {
return finish( return finish(
await executeCloudAgentTool( await executeCloudAgentTool(
@ -3824,6 +3841,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
pluginLine, pluginLine,
agentSkillsLine, agentSkillsLine,
taughtSkillsLine, taughtSkillsLine,
exposedToolNames.has("media_search")
? "For photo and video work, read the Video Producer skill with skill_read before using the media_* tools."
: undefined,
exposedToolNames.has("render_plot") exposedToolNames.has("render_plot")
? 'For charts and data visualization, use the render_plot tool: it renders bar, line, scatter, histogram, heatmap, faceted and many more chart types from a JSON spec and attaches the PNG to the chat. Call render_plot with {"help": true} before your first chart to read the full guide.' ? 'For charts and data visualization, use the render_plot tool: it renders bar, line, scatter, histogram, heatmap, faceted and many more chart types from a JSON spec and attaches the PNG to the chat. Call render_plot with {"help": true} before your first chart to read the full guide.'
: undefined, : undefined,

View File

@ -118,6 +118,7 @@ export {
} from "./media-render.js"; } from "./media-render.js";
export * from "./media-services.js"; export * from "./media-services.js";
export * from "./media-sidecar.js"; export * from "./media-sidecar.js";
export * from "./media-skills.js";
export * from "./media-zip.js"; export * from "./media-zip.js";
export * from "./memory-provider-factory.js"; export * from "./memory-provider-factory.js";
export * from "./messaging-context.js"; export * from "./messaging-context.js";

View File

@ -18,6 +18,7 @@ import { MODEL_TIERS, ModelTierSchema } from "@rakazo/contracts";
import { getLogger } from "@rakazo/logging"; import { getLogger } from "@rakazo/logging";
import { isToolPauseResult } from "./approval-effect.js"; import { isToolPauseResult } from "./approval-effect.js";
import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js"; import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js";
import { describeMediaToolActivity } from "./media-tools.js";
import { import {
normalizeOpenAiToolParameters, normalizeOpenAiToolParameters,
openAiToolParametersNeedNormalization, openAiToolParametersNeedNormalization,
@ -474,6 +475,8 @@ export function describeToolActivity(toolName: string, args: unknown): string {
if (toolName === "attach_file") return `Attaching ${detail(record.path)}`; if (toolName === "attach_file") return `Attaching ${detail(record.path)}`;
if (toolName === "open_path") return `Opening ${detail(record.path)}`; if (toolName === "open_path") return `Opening ${detail(record.path)}`;
if (toolName === "render_plot") return "Rendering a chart"; if (toolName === "render_plot") return "Rendering a chart";
const media = describeMediaToolActivity(toolName, record);
if (media) return media;
if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`; if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`;
if (toolName === "computer_observe") return "Looking at the screen"; if (toolName === "computer_observe") return "Looking at the screen";
if (toolName === "browser_navigate") if (toolName === "browser_navigate")