feat(media): wire agent tools and skills, ffmpeg in compose image
Hook the media_* tools and the Video Producer and Video Copy skills into the executor, pi-runtime activity labels and the skill catalog; all stay hidden unless Media Studio is configured. Install ffmpeg in the compose image for renders and drop a local path from ARCHITECTURE.md. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w
This commit is contained in:
parent
80f082f910
commit
e8f8912c65
@ -3,7 +3,7 @@
|
|||||||
> **Status**: RATIFIZIERT & AKTIV
|
> **Status**: RATIFIZIERT & AKTIV
|
||||||
> **Ecosystem**: evOo Autonomous Agent Platform
|
> **Ecosystem**: evOo Autonomous Agent Platform
|
||||||
> **Executive Lead / CTO**: Steve Jobs (`templates/cto-steve-jobs`)
|
> **Executive Lead / CTO**: Steve Jobs (`templates/cto-steve-jobs`)
|
||||||
> **Konfigurationsanker**: [`evoo.config.json`](file:///Users/ingowagner/Projects/evOo-local/evoo.config.json)
|
> **Konfigurationsanker**: [`evoo.config.json`](evoo.config.json)
|
||||||
> **Grundsatz**: *Realität im Dateisystem vor Halluzination im Kopf. Nur was physisch geschrieben, getestet und lauffähig ist, existiert.*
|
> **Grundsatz**: *Realität im Dateisystem vor Halluzination im Kopf. Nur was physisch geschrieben, getestet und lauffähig ist, existiert.*
|
||||||
|
|
||||||
---
|
---
|
||||||
@ -62,7 +62,7 @@
|
|||||||
Das evOo-Ökosystem ist in klar getrennte, orthogonale Säulen unterteilt:
|
Das evOo-Ökosystem ist in klar getrennte, orthogonale Säulen unterteilt:
|
||||||
|
|
||||||
```
|
```
|
||||||
/Users/ingowagner/Projects/evOo-local/
|
evOo-local/
|
||||||
├── ARCHITECTURE.md # Diese Verfassung
|
├── ARCHITECTURE.md # Diese Verfassung
|
||||||
├── evoo.config.json # Autonome Berechtigungen & Policies
|
├── evoo.config.json # Autonome Berechtigungen & Policies
|
||||||
├── deploy.sh # Hot-Reload, Health-Checks & Service-Orchestrierung
|
├── deploy.sh # Hot-Reload, Health-Checks & Service-Orchestrierung
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
import { ORPCError } from "@orpc/server";
|
import { ORPCError } from "@orpc/server";
|
||||||
import { BUILTIN_AGENT_SKILLS } from "@rakazo/adapters";
|
import { BUILTIN_AGENT_SKILLS, filterMediaSkills } from "@rakazo/adapters";
|
||||||
import type { Actor, AgentSkill, AgentSkillSource } from "@rakazo/contracts";
|
import type { Actor, AgentSkill, AgentSkillSource } from "@rakazo/contracts";
|
||||||
import {
|
import {
|
||||||
buildSkillMd,
|
buildSkillMd,
|
||||||
@ -118,7 +118,13 @@ export function resolveSkillContent(input: {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createAgentSkillsService(prisma: PrismaClient) {
|
export function createAgentSkillsService(
|
||||||
|
prisma: PrismaClient,
|
||||||
|
options: { mediaConfigured?: boolean } = {},
|
||||||
|
) {
|
||||||
|
/** Built-in skills this deployment can run: media recipes only with Media Studio. */
|
||||||
|
const visibleBuiltins = () =>
|
||||||
|
filterMediaSkills(builtinCatalog(), Boolean(options.mediaConfigured));
|
||||||
async function owned(actor: Actor, skillId: string) {
|
async function owned(actor: Actor, skillId: string) {
|
||||||
const row = await prisma.agentSkill.findFirst({
|
const row = await prisma.agentSkill.findFirst({
|
||||||
where: {
|
where: {
|
||||||
@ -137,7 +143,7 @@ export function createAgentSkillsService(prisma: PrismaClient) {
|
|||||||
where: { spaceId: actor.spaceId, userId: actor.userId },
|
where: { spaceId: actor.spaceId, userId: actor.userId },
|
||||||
orderBy: [{ name: "asc" }, { id: "asc" }],
|
orderBy: [{ name: "asc" }, { id: "asc" }],
|
||||||
});
|
});
|
||||||
return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill)).map(
|
return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill)).map(
|
||||||
({ content: _content, ...entry }) => entry,
|
({ content: _content, ...entry }) => entry,
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
@ -147,12 +153,12 @@ export function createAgentSkillsService(prisma: PrismaClient) {
|
|||||||
where: { spaceId: actor.spaceId, userId: actor.userId },
|
where: { spaceId: actor.spaceId, userId: actor.userId },
|
||||||
orderBy: [{ name: "asc" }, { id: "asc" }],
|
orderBy: [{ name: "asc" }, { id: "asc" }],
|
||||||
});
|
});
|
||||||
return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill));
|
return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill));
|
||||||
},
|
},
|
||||||
|
|
||||||
async get(actor: Actor, input: { skillId?: string; name?: string }): Promise<AgentSkill> {
|
async get(actor: Actor, input: { skillId?: string; name?: string }): Promise<AgentSkill> {
|
||||||
if (input.skillId?.startsWith("builtin:")) {
|
if (input.skillId?.startsWith("builtin:")) {
|
||||||
const builtin = builtinCatalog().find((skill) => skill.id === input.skillId);
|
const builtin = visibleBuiltins().find((skill) => skill.id === input.skillId);
|
||||||
if (!builtin) throw new IsolationError();
|
if (!builtin) throw new IsolationError();
|
||||||
return builtin;
|
return builtin;
|
||||||
}
|
}
|
||||||
@ -169,7 +175,7 @@ export function createAgentSkillsService(prisma: PrismaClient) {
|
|||||||
});
|
});
|
||||||
const row = findSkillByName(rows, name);
|
const row = findSkillByName(rows, name);
|
||||||
if (row) return mapAgentSkill(row);
|
if (row) return mapAgentSkill(row);
|
||||||
const builtin = findSkillByName(builtinCatalog(), name);
|
const builtin = findSkillByName(visibleBuiltins(), name);
|
||||||
if (!builtin) throw new IsolationError();
|
if (!builtin) throw new IsolationError();
|
||||||
return builtin;
|
return builtin;
|
||||||
},
|
},
|
||||||
|
|||||||
@ -415,6 +415,8 @@ export async function createApp(
|
|||||||
web: createWebProvider(),
|
web: createWebProvider(),
|
||||||
cloudAgent,
|
cloudAgent,
|
||||||
shutdownSignal: shutdown.signal,
|
shutdownSignal: shutdown.signal,
|
||||||
|
// Late-bound: media services are built below and need executor.resolveModel.
|
||||||
|
media: () => mediaServices,
|
||||||
});
|
});
|
||||||
|
|
||||||
const mediaServices = createMediaServices({
|
const mediaServices = createMediaServices({
|
||||||
|
|||||||
@ -522,7 +522,9 @@ export function createRouter(deps: RouterDeps) {
|
|||||||
home: deps.home,
|
home: deps.home,
|
||||||
dataDir: deps.dataDir,
|
dataDir: deps.dataDir,
|
||||||
});
|
});
|
||||||
const agentSkills = createAgentSkillsService(deps.prisma);
|
const agentSkills = createAgentSkillsService(deps.prisma, {
|
||||||
|
mediaConfigured: Boolean(deps.mediaServices),
|
||||||
|
});
|
||||||
|
|
||||||
const authed = os.use(async ({ context, next }) => {
|
const authed = os.use(async ({ context, next }) => {
|
||||||
if (!context.actor) throw new ORPCError("UNAUTHORIZED");
|
if (!context.actor) throw new ORPCError("UNAUTHORIZED");
|
||||||
|
|||||||
@ -202,6 +202,8 @@ async function main() {
|
|||||||
messaging: messaging ? createMessagingContextLoader(prisma) : undefined,
|
messaging: messaging ? createMessagingContextLoader(prisma) : undefined,
|
||||||
web: createWebProvider(),
|
web: createWebProvider(),
|
||||||
cloudAgent,
|
cloudAgent,
|
||||||
|
// Late-bound: media services are built below and need executor.resolveModel.
|
||||||
|
media: () => mediaServices,
|
||||||
});
|
});
|
||||||
|
|
||||||
const mediaAbort = new AbortController();
|
const mediaAbort = new AbortController();
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a
|
FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a
|
||||||
RUN corepack enable
|
RUN corepack enable
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
# Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise
|
# Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise
|
||||||
# leave the copied tree unreadable to USER node at runtime.
|
# leave the copied tree unreadable to USER node at runtime.
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
import { buildSkillMd } from "@rakazo/core";
|
import { buildSkillMd } from "@rakazo/core";
|
||||||
|
import { MEDIA_AGENT_SKILLS } from "./media-skills.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Built-in Agent Skills (SKILL.md recipes) available to every user.
|
* Built-in Agent Skills (SKILL.md recipes) available to every user.
|
||||||
@ -21,6 +22,7 @@ const SKILLS: Array<{ name: string; description: string; body: string }> = [
|
|||||||
3. Verify before accusing: for each suspected issue, re-read the code and construct the concrete input or state that triggers the failure. Drop anything you cannot substantiate.
|
3. Verify before accusing: for each suspected issue, re-read the code and construct the concrete input or state that triggers the failure. Drop anything you cannot substantiate.
|
||||||
4. Synthesize a verdict: ship, ship after fixes, or do not ship. List the confirmed findings ordered by severity, each with its location and failure scenario, then any open questions. If nothing survived verification, say so plainly instead of inventing nitpicks.`,
|
4. Synthesize a verdict: ship, ship after fixes, or do not ship. List the confirmed findings ordered by severity, each with its location and failure scenario, then any open questions. If nothing survived verification, say so plainly instead of inventing nitpicks.`,
|
||||||
},
|
},
|
||||||
|
...MEDIA_AGENT_SKILLS,
|
||||||
];
|
];
|
||||||
|
|
||||||
export const BUILTIN_AGENT_SKILLS: Array<{
|
export const BUILTIN_AGENT_SKILLS: Array<{
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import {
|
|||||||
USER_FACT_CATEGORIES,
|
USER_FACT_CATEGORIES,
|
||||||
} from "@rakazo/contracts";
|
} from "@rakazo/contracts";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
import { mediaAgentTools } from "./media-tools.js";
|
||||||
|
|
||||||
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
||||||
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
||||||
@ -963,6 +964,8 @@ export const builtinAgentTools: ConnectorTool[] = [
|
|||||||
required: ["bot", "title", "brief"],
|
required: ["bot", "title", "brief"],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
// Offered only when Media Studio is configured (filterMediaTools in the executor).
|
||||||
|
...mediaAgentTools,
|
||||||
];
|
];
|
||||||
|
|
||||||
/** Agent-connection tools, exposed only when the messaging surface is enabled. */
|
/** Agent-connection tools, exposed only when the messaging surface is enabled. */
|
||||||
|
|||||||
@ -13,6 +13,7 @@ import type {
|
|||||||
} from "@rakazo/adapter-kit";
|
} from "@rakazo/adapter-kit";
|
||||||
import type { PrismaClient, ThreadEvents } from "@rakazo/db";
|
import type { PrismaClient, ThreadEvents } from "@rakazo/db";
|
||||||
import type { CloudAgentConnection } from "./cloud-agent-factory.js";
|
import type { CloudAgentConnection } from "./cloud-agent-factory.js";
|
||||||
|
import type { MediaServices } from "./media-context.js";
|
||||||
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
|
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
|
||||||
import type { RemoteTransportDependencies } from "./remote-mcp.js";
|
import type { RemoteTransportDependencies } from "./remote-mcp.js";
|
||||||
import type { EncryptedSecretStore } from "./secrets.js";
|
import type { EncryptedSecretStore } from "./secrets.js";
|
||||||
@ -45,6 +46,8 @@ export interface ExecutorDeps {
|
|||||||
secretHttp?: RemoteTransportDependencies;
|
secretHttp?: RemoteTransportDependencies;
|
||||||
/** Remote cloud coding agents. Null/omit means tools stay uninjected. */
|
/** Remote cloud coding agents. Null/omit means tools stay uninjected. */
|
||||||
cloudAgent?: CloudAgentConnection | null;
|
cloudAgent?: CloudAgentConnection | null;
|
||||||
|
/** Media Studio services, read per run; null or omitted keeps the media_* tools hidden. */
|
||||||
|
media?: () => MediaServices | null;
|
||||||
/** Aborted when createApp stop() begins so in-flight continueRun boot waits exit promptly. */
|
/** Aborted when createApp stop() begins so in-flight continueRun boot waits exit promptly. */
|
||||||
shutdownSignal?: AbortSignal;
|
shutdownSignal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -290,6 +290,9 @@ import {
|
|||||||
needsOAuthProbe,
|
needsOAuthProbe,
|
||||||
parseMcpServerToolArgs,
|
parseMcpServerToolArgs,
|
||||||
} from "./mcp-server-tool.js";
|
} from "./mcp-server-tool.js";
|
||||||
|
import { filterMediaSkills } from "./media-skills.js";
|
||||||
|
import { filterMediaTools, isMediaToolName } from "./media-tools.js";
|
||||||
|
import { executeMediaTool } from "./media-tools-execute.js";
|
||||||
import { loadAgentMemoryContext } from "./memory-context.js";
|
import { loadAgentMemoryContext } from "./memory-context.js";
|
||||||
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
|
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
|
||||||
import { selectMemoryTools } from "./memory-tools.js";
|
import { selectMemoryTools } from "./memory-tools.js";
|
||||||
@ -1127,7 +1130,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
|||||||
listAgentSkillRecords(deps.prisma, {
|
listAgentSkillRecords(deps.prisma, {
|
||||||
spaceId: run.spaceId,
|
spaceId: run.spaceId,
|
||||||
userId: run.userId,
|
userId: run.userId,
|
||||||
}),
|
}).then((records) => filterMediaSkills(records, Boolean(deps.media?.()))),
|
||||||
deps.prisma.agentSecret.findMany({
|
deps.prisma.agentSecret.findMany({
|
||||||
where: agentSecretScopeForRun(run.spaceId, run.userId),
|
where: agentSecretScopeForRun(run.spaceId, run.userId),
|
||||||
select: {
|
select: {
|
||||||
@ -1477,18 +1480,22 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
|||||||
select: { id: true },
|
select: { id: true },
|
||||||
}))
|
}))
|
||||||
: false;
|
: false;
|
||||||
const builtins = selectBuiltinToolsForRun({
|
const mediaServices = deps.media?.() ?? null;
|
||||||
graphicalToolsAllowed,
|
const builtins = filterMediaTools(
|
||||||
pageBrowserAllowed,
|
selectBuiltinToolsForRun({
|
||||||
groupId: thread.groupId,
|
graphicalToolsAllowed,
|
||||||
trigger: run.trigger,
|
pageBrowserAllowed,
|
||||||
semanticMemoryEnabled,
|
groupId: thread.groupId,
|
||||||
cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId),
|
trigger: run.trigger,
|
||||||
messagingChannelRun,
|
semanticMemoryEnabled,
|
||||||
agentConnectionsEnabled: hasMessagingIdentity,
|
cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId),
|
||||||
foreignAgentRun,
|
messagingChannelRun,
|
||||||
bot,
|
agentConnectionsEnabled: hasMessagingIdentity,
|
||||||
});
|
foreignAgentRun,
|
||||||
|
bot,
|
||||||
|
}),
|
||||||
|
mediaServices !== null,
|
||||||
|
);
|
||||||
const exposedConnectorTools = discovered.filter(
|
const exposedConnectorTools = discovered.filter(
|
||||||
(tool) => !builtinAgentTools.some((builtin) => builtin.name === tool.name),
|
(tool) => !builtinAgentTools.some((builtin) => builtin.name === tool.name),
|
||||||
);
|
);
|
||||||
@ -2635,6 +2642,16 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isMediaToolName(name)) {
|
||||||
|
return finish(
|
||||||
|
await executeMediaTool(
|
||||||
|
mediaServices,
|
||||||
|
{ spaceId: run.spaceId, userId: run.userId },
|
||||||
|
name,
|
||||||
|
args,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
if (name.startsWith("cloud_agent_")) {
|
if (name.startsWith("cloud_agent_")) {
|
||||||
return finish(
|
return finish(
|
||||||
await executeCloudAgentTool(
|
await executeCloudAgentTool(
|
||||||
@ -3824,6 +3841,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
|||||||
pluginLine,
|
pluginLine,
|
||||||
agentSkillsLine,
|
agentSkillsLine,
|
||||||
taughtSkillsLine,
|
taughtSkillsLine,
|
||||||
|
exposedToolNames.has("media_search")
|
||||||
|
? "For photo and video work, read the Video Producer skill with skill_read before using the media_* tools."
|
||||||
|
: undefined,
|
||||||
exposedToolNames.has("render_plot")
|
exposedToolNames.has("render_plot")
|
||||||
? 'For charts and data visualization, use the render_plot tool: it renders bar, line, scatter, histogram, heatmap, faceted and many more chart types from a JSON spec and attaches the PNG to the chat. Call render_plot with {"help": true} before your first chart to read the full guide.'
|
? 'For charts and data visualization, use the render_plot tool: it renders bar, line, scatter, histogram, heatmap, faceted and many more chart types from a JSON spec and attaches the PNG to the chat. Call render_plot with {"help": true} before your first chart to read the full guide.'
|
||||||
: undefined,
|
: undefined,
|
||||||
|
|||||||
@ -118,6 +118,7 @@ export {
|
|||||||
} from "./media-render.js";
|
} from "./media-render.js";
|
||||||
export * from "./media-services.js";
|
export * from "./media-services.js";
|
||||||
export * from "./media-sidecar.js";
|
export * from "./media-sidecar.js";
|
||||||
|
export * from "./media-skills.js";
|
||||||
export * from "./media-zip.js";
|
export * from "./media-zip.js";
|
||||||
export * from "./memory-provider-factory.js";
|
export * from "./memory-provider-factory.js";
|
||||||
export * from "./messaging-context.js";
|
export * from "./messaging-context.js";
|
||||||
|
|||||||
@ -18,6 +18,7 @@ import { MODEL_TIERS, ModelTierSchema } from "@rakazo/contracts";
|
|||||||
import { getLogger } from "@rakazo/logging";
|
import { getLogger } from "@rakazo/logging";
|
||||||
import { isToolPauseResult } from "./approval-effect.js";
|
import { isToolPauseResult } from "./approval-effect.js";
|
||||||
import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js";
|
import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js";
|
||||||
|
import { describeMediaToolActivity } from "./media-tools.js";
|
||||||
import {
|
import {
|
||||||
normalizeOpenAiToolParameters,
|
normalizeOpenAiToolParameters,
|
||||||
openAiToolParametersNeedNormalization,
|
openAiToolParametersNeedNormalization,
|
||||||
@ -474,6 +475,8 @@ export function describeToolActivity(toolName: string, args: unknown): string {
|
|||||||
if (toolName === "attach_file") return `Attaching ${detail(record.path)}`;
|
if (toolName === "attach_file") return `Attaching ${detail(record.path)}`;
|
||||||
if (toolName === "open_path") return `Opening ${detail(record.path)}`;
|
if (toolName === "open_path") return `Opening ${detail(record.path)}`;
|
||||||
if (toolName === "render_plot") return "Rendering a chart";
|
if (toolName === "render_plot") return "Rendering a chart";
|
||||||
|
const media = describeMediaToolActivity(toolName, record);
|
||||||
|
if (media) return media;
|
||||||
if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`;
|
if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`;
|
||||||
if (toolName === "computer_observe") return "Looking at the screen";
|
if (toolName === "computer_observe") return "Looking at the screen";
|
||||||
if (toolName === "browser_navigate")
|
if (toolName === "browser_navigate")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user