feat(media): wire agent tools and skills, ffmpeg in compose image

Hook the media_* tools and the Video Producer and Video Copy skills into the
executor, pi-runtime activity labels and the skill catalog; all stay hidden
unless Media Studio is configured. Install ffmpeg in the compose image for
renders and drop a local path from ARCHITECTURE.md.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w
This commit is contained in:
Claude 2026-10-03 19:21:45 +00:00
parent 80f082f910
commit e8f8912c65
No known key found for this signature in database
12 changed files with 67 additions and 22 deletions

View File

@ -3,7 +3,7 @@
> **Status**: RATIFIZIERT & AKTIV
> **Ecosystem**: evOo Autonomous Agent Platform
> **Executive Lead / CTO**: Steve Jobs (`templates/cto-steve-jobs`)
> **Konfigurationsanker**: [`evoo.config.json`](file:///Users/ingowagner/Projects/evOo-local/evoo.config.json)
> **Konfigurationsanker**: [`evoo.config.json`](evoo.config.json)
> **Grundsatz**: *Realität im Dateisystem vor Halluzination im Kopf. Nur was physisch geschrieben, getestet und lauffähig ist, existiert.*
---
@ -62,7 +62,7 @@
Das evOo-Ökosystem ist in klar getrennte, orthogonale Säulen unterteilt:
```
/Users/ingowagner/Projects/evOo-local/
evOo-local/
├── ARCHITECTURE.md # Diese Verfassung
├── evoo.config.json # Autonome Berechtigungen & Policies
├── deploy.sh # Hot-Reload, Health-Checks & Service-Orchestrierung

View File

@ -1,5 +1,5 @@
import { ORPCError } from "@orpc/server";
import { BUILTIN_AGENT_SKILLS } from "@rakazo/adapters";
import { BUILTIN_AGENT_SKILLS, filterMediaSkills } from "@rakazo/adapters";
import type { Actor, AgentSkill, AgentSkillSource } from "@rakazo/contracts";
import {
buildSkillMd,
@ -118,7 +118,13 @@ export function resolveSkillContent(input: {
};
}
export function createAgentSkillsService(prisma: PrismaClient) {
export function createAgentSkillsService(
prisma: PrismaClient,
options: { mediaConfigured?: boolean } = {},
) {
/** Built-in skills this deployment can run: media recipes only with Media Studio. */
const visibleBuiltins = () =>
filterMediaSkills(builtinCatalog(), Boolean(options.mediaConfigured));
async function owned(actor: Actor, skillId: string) {
const row = await prisma.agentSkill.findFirst({
where: {
@ -137,7 +143,7 @@ export function createAgentSkillsService(prisma: PrismaClient) {
where: { spaceId: actor.spaceId, userId: actor.userId },
orderBy: [{ name: "asc" }, { id: "asc" }],
});
return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill)).map(
return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill)).map(
({ content: _content, ...entry }) => entry,
);
},
@ -147,12 +153,12 @@ export function createAgentSkillsService(prisma: PrismaClient) {
where: { spaceId: actor.spaceId, userId: actor.userId },
orderBy: [{ name: "asc" }, { id: "asc" }],
});
return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill));
return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill));
},
async get(actor: Actor, input: { skillId?: string; name?: string }): Promise<AgentSkill> {
if (input.skillId?.startsWith("builtin:")) {
const builtin = builtinCatalog().find((skill) => skill.id === input.skillId);
const builtin = visibleBuiltins().find((skill) => skill.id === input.skillId);
if (!builtin) throw new IsolationError();
return builtin;
}
@ -169,7 +175,7 @@ export function createAgentSkillsService(prisma: PrismaClient) {
});
const row = findSkillByName(rows, name);
if (row) return mapAgentSkill(row);
const builtin = findSkillByName(builtinCatalog(), name);
const builtin = findSkillByName(visibleBuiltins(), name);
if (!builtin) throw new IsolationError();
return builtin;
},

View File

@ -415,6 +415,8 @@ export async function createApp(
web: createWebProvider(),
cloudAgent,
shutdownSignal: shutdown.signal,
// Late-bound: media services are built below and need executor.resolveModel.
media: () => mediaServices,
});
const mediaServices = createMediaServices({

View File

@ -522,7 +522,9 @@ export function createRouter(deps: RouterDeps) {
home: deps.home,
dataDir: deps.dataDir,
});
const agentSkills = createAgentSkillsService(deps.prisma);
const agentSkills = createAgentSkillsService(deps.prisma, {
mediaConfigured: Boolean(deps.mediaServices),
});
const authed = os.use(async ({ context, next }) => {
if (!context.actor) throw new ORPCError("UNAUTHORIZED");

View File

@ -202,6 +202,8 @@ async function main() {
messaging: messaging ? createMessagingContextLoader(prisma) : undefined,
web: createWebProvider(),
cloudAgent,
// Late-bound: media services are built below and need executor.resolveModel.
media: () => mediaServices,
});
const mediaAbort = new AbortController();

View File

@ -1,5 +1,6 @@
FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a
RUN corepack enable
RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise
# leave the copied tree unreadable to USER node at runtime.

View File

@ -1,4 +1,5 @@
import { buildSkillMd } from "@rakazo/core";
import { MEDIA_AGENT_SKILLS } from "./media-skills.js";
/**
* Built-in Agent Skills (SKILL.md recipes) available to every user.
@ -21,6 +22,7 @@ const SKILLS: Array<{ name: string; description: string; body: string }> = [
3. Verify before accusing: for each suspected issue, re-read the code and construct the concrete input or state that triggers the failure. Drop anything you cannot substantiate.
4. Synthesize a verdict: ship, ship after fixes, or do not ship. List the confirmed findings ordered by severity, each with its location and failure scenario, then any open questions. If nothing survived verification, say so plainly instead of inventing nitpicks.`,
},
...MEDIA_AGENT_SKILLS,
];
export const BUILTIN_AGENT_SKILLS: Array<{

View File

@ -8,6 +8,7 @@ import {
USER_FACT_CATEGORIES,
} from "@rakazo/contracts";
import { z } from "zod";
import { mediaAgentTools } from "./media-tools.js";
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
@ -963,6 +964,8 @@ export const builtinAgentTools: ConnectorTool[] = [
required: ["bot", "title", "brief"],
},
},
// Offered only when Media Studio is configured (filterMediaTools in the executor).
...mediaAgentTools,
];
/** Agent-connection tools, exposed only when the messaging surface is enabled. */

View File

@ -13,6 +13,7 @@ import type {
} from "@rakazo/adapter-kit";
import type { PrismaClient, ThreadEvents } from "@rakazo/db";
import type { CloudAgentConnection } from "./cloud-agent-factory.js";
import type { MediaServices } from "./media-context.js";
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
import type { RemoteTransportDependencies } from "./remote-mcp.js";
import type { EncryptedSecretStore } from "./secrets.js";
@ -45,6 +46,8 @@ export interface ExecutorDeps {
secretHttp?: RemoteTransportDependencies;
/** Remote cloud coding agents. Null/omit means tools stay uninjected. */
cloudAgent?: CloudAgentConnection | null;
/** Media Studio services, read per run; null or omitted keeps the media_* tools hidden. */
media?: () => MediaServices | null;
/** Aborted when createApp stop() begins so in-flight continueRun boot waits exit promptly. */
shutdownSignal?: AbortSignal;
}

View File

@ -290,6 +290,9 @@ import {
needsOAuthProbe,
parseMcpServerToolArgs,
} from "./mcp-server-tool.js";
import { filterMediaSkills } from "./media-skills.js";
import { filterMediaTools, isMediaToolName } from "./media-tools.js";
import { executeMediaTool } from "./media-tools-execute.js";
import { loadAgentMemoryContext } from "./memory-context.js";
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
import { selectMemoryTools } from "./memory-tools.js";
@ -1127,7 +1130,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
listAgentSkillRecords(deps.prisma, {
spaceId: run.spaceId,
userId: run.userId,
}),
}).then((records) => filterMediaSkills(records, Boolean(deps.media?.()))),
deps.prisma.agentSecret.findMany({
where: agentSecretScopeForRun(run.spaceId, run.userId),
select: {
@ -1477,7 +1480,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
select: { id: true },
}))
: false;
const builtins = selectBuiltinToolsForRun({
const mediaServices = deps.media?.() ?? null;
const builtins = filterMediaTools(
selectBuiltinToolsForRun({
graphicalToolsAllowed,
pageBrowserAllowed,
groupId: thread.groupId,
@ -1488,7 +1493,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
agentConnectionsEnabled: hasMessagingIdentity,
foreignAgentRun,
bot,
});
}),
mediaServices !== null,
);
const exposedConnectorTools = discovered.filter(
(tool) => !builtinAgentTools.some((builtin) => builtin.name === tool.name),
);
@ -2635,6 +2642,16 @@ export function createRunExecutor(deps: ExecutorDeps) {
);
}
if (isMediaToolName(name)) {
return finish(
await executeMediaTool(
mediaServices,
{ spaceId: run.spaceId, userId: run.userId },
name,
args,
),
);
}
if (name.startsWith("cloud_agent_")) {
return finish(
await executeCloudAgentTool(
@ -3824,6 +3841,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
pluginLine,
agentSkillsLine,
taughtSkillsLine,
exposedToolNames.has("media_search")
? "For photo and video work, read the Video Producer skill with skill_read before using the media_* tools."
: undefined,
exposedToolNames.has("render_plot")
? 'For charts and data visualization, use the render_plot tool: it renders bar, line, scatter, histogram, heatmap, faceted and many more chart types from a JSON spec and attaches the PNG to the chat. Call render_plot with {"help": true} before your first chart to read the full guide.'
: undefined,

View File

@ -118,6 +118,7 @@ export {
} from "./media-render.js";
export * from "./media-services.js";
export * from "./media-sidecar.js";
export * from "./media-skills.js";
export * from "./media-zip.js";
export * from "./memory-provider-factory.js";
export * from "./messaging-context.js";

View File

@ -18,6 +18,7 @@ import { MODEL_TIERS, ModelTierSchema } from "@rakazo/contracts";
import { getLogger } from "@rakazo/logging";
import { isToolPauseResult } from "./approval-effect.js";
import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js";
import { describeMediaToolActivity } from "./media-tools.js";
import {
normalizeOpenAiToolParameters,
openAiToolParametersNeedNormalization,
@ -474,6 +475,8 @@ export function describeToolActivity(toolName: string, args: unknown): string {
if (toolName === "attach_file") return `Attaching ${detail(record.path)}`;
if (toolName === "open_path") return `Opening ${detail(record.path)}`;
if (toolName === "render_plot") return "Rendering a chart";
const media = describeMediaToolActivity(toolName, record);
if (media) return media;
if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`;
if (toolName === "computer_observe") return "Looking at the screen";
if (toolName === "browser_navigate")