feat(media): wire agent tools and skills, ffmpeg in compose image
Hook the media_* tools and the Video Producer and Video Copy skills into the executor, pi-runtime activity labels and the skill catalog; all stay hidden unless Media Studio is configured. Install ffmpeg in the compose image for renders and drop a local path from ARCHITECTURE.md. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w
This commit is contained in:
parent
80f082f910
commit
e8f8912c65
@ -3,7 +3,7 @@
|
||||
> **Status**: RATIFIZIERT & AKTIV
|
||||
> **Ecosystem**: evOo Autonomous Agent Platform
|
||||
> **Executive Lead / CTO**: Steve Jobs (`templates/cto-steve-jobs`)
|
||||
> **Konfigurationsanker**: [`evoo.config.json`](file:///Users/ingowagner/Projects/evOo-local/evoo.config.json)
|
||||
> **Konfigurationsanker**: [`evoo.config.json`](evoo.config.json)
|
||||
> **Grundsatz**: *Realität im Dateisystem vor Halluzination im Kopf. Nur was physisch geschrieben, getestet und lauffähig ist, existiert.*
|
||||
|
||||
---
|
||||
@ -62,7 +62,7 @@
|
||||
Das evOo-Ökosystem ist in klar getrennte, orthogonale Säulen unterteilt:
|
||||
|
||||
```
|
||||
/Users/ingowagner/Projects/evOo-local/
|
||||
evOo-local/
|
||||
├── ARCHITECTURE.md # Diese Verfassung
|
||||
├── evoo.config.json # Autonome Berechtigungen & Policies
|
||||
├── deploy.sh # Hot-Reload, Health-Checks & Service-Orchestrierung
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { ORPCError } from "@orpc/server";
|
||||
import { BUILTIN_AGENT_SKILLS } from "@rakazo/adapters";
|
||||
import { BUILTIN_AGENT_SKILLS, filterMediaSkills } from "@rakazo/adapters";
|
||||
import type { Actor, AgentSkill, AgentSkillSource } from "@rakazo/contracts";
|
||||
import {
|
||||
buildSkillMd,
|
||||
@ -118,7 +118,13 @@ export function resolveSkillContent(input: {
|
||||
};
|
||||
}
|
||||
|
||||
export function createAgentSkillsService(prisma: PrismaClient) {
|
||||
export function createAgentSkillsService(
|
||||
prisma: PrismaClient,
|
||||
options: { mediaConfigured?: boolean } = {},
|
||||
) {
|
||||
/** Built-in skills this deployment can run: media recipes only with Media Studio. */
|
||||
const visibleBuiltins = () =>
|
||||
filterMediaSkills(builtinCatalog(), Boolean(options.mediaConfigured));
|
||||
async function owned(actor: Actor, skillId: string) {
|
||||
const row = await prisma.agentSkill.findFirst({
|
||||
where: {
|
||||
@ -137,7 +143,7 @@ export function createAgentSkillsService(prisma: PrismaClient) {
|
||||
where: { spaceId: actor.spaceId, userId: actor.userId },
|
||||
orderBy: [{ name: "asc" }, { id: "asc" }],
|
||||
});
|
||||
return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill)).map(
|
||||
return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill)).map(
|
||||
({ content: _content, ...entry }) => entry,
|
||||
);
|
||||
},
|
||||
@ -147,12 +153,12 @@ export function createAgentSkillsService(prisma: PrismaClient) {
|
||||
where: { spaceId: actor.spaceId, userId: actor.userId },
|
||||
orderBy: [{ name: "asc" }, { id: "asc" }],
|
||||
});
|
||||
return mergeBuiltinSkills(builtinCatalog(), rows.map(mapAgentSkill));
|
||||
return mergeBuiltinSkills(visibleBuiltins(), rows.map(mapAgentSkill));
|
||||
},
|
||||
|
||||
async get(actor: Actor, input: { skillId?: string; name?: string }): Promise<AgentSkill> {
|
||||
if (input.skillId?.startsWith("builtin:")) {
|
||||
const builtin = builtinCatalog().find((skill) => skill.id === input.skillId);
|
||||
const builtin = visibleBuiltins().find((skill) => skill.id === input.skillId);
|
||||
if (!builtin) throw new IsolationError();
|
||||
return builtin;
|
||||
}
|
||||
@ -169,7 +175,7 @@ export function createAgentSkillsService(prisma: PrismaClient) {
|
||||
});
|
||||
const row = findSkillByName(rows, name);
|
||||
if (row) return mapAgentSkill(row);
|
||||
const builtin = findSkillByName(builtinCatalog(), name);
|
||||
const builtin = findSkillByName(visibleBuiltins(), name);
|
||||
if (!builtin) throw new IsolationError();
|
||||
return builtin;
|
||||
},
|
||||
|
||||
@ -415,6 +415,8 @@ export async function createApp(
|
||||
web: createWebProvider(),
|
||||
cloudAgent,
|
||||
shutdownSignal: shutdown.signal,
|
||||
// Late-bound: media services are built below and need executor.resolveModel.
|
||||
media: () => mediaServices,
|
||||
});
|
||||
|
||||
const mediaServices = createMediaServices({
|
||||
|
||||
@ -522,7 +522,9 @@ export function createRouter(deps: RouterDeps) {
|
||||
home: deps.home,
|
||||
dataDir: deps.dataDir,
|
||||
});
|
||||
const agentSkills = createAgentSkillsService(deps.prisma);
|
||||
const agentSkills = createAgentSkillsService(deps.prisma, {
|
||||
mediaConfigured: Boolean(deps.mediaServices),
|
||||
});
|
||||
|
||||
const authed = os.use(async ({ context, next }) => {
|
||||
if (!context.actor) throw new ORPCError("UNAUTHORIZED");
|
||||
|
||||
@ -202,6 +202,8 @@ async function main() {
|
||||
messaging: messaging ? createMessagingContextLoader(prisma) : undefined,
|
||||
web: createWebProvider(),
|
||||
cloudAgent,
|
||||
// Late-bound: media services are built below and need executor.resolveModel.
|
||||
media: () => mediaServices,
|
||||
});
|
||||
|
||||
const mediaAbort = new AbortController();
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a
|
||||
RUN corepack enable
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
# Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise
|
||||
# leave the copied tree unreadable to USER node at runtime.
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { buildSkillMd } from "@rakazo/core";
|
||||
import { MEDIA_AGENT_SKILLS } from "./media-skills.js";
|
||||
|
||||
/**
|
||||
* Built-in Agent Skills (SKILL.md recipes) available to every user.
|
||||
@ -21,6 +22,7 @@ const SKILLS: Array<{ name: string; description: string; body: string }> = [
|
||||
3. Verify before accusing: for each suspected issue, re-read the code and construct the concrete input or state that triggers the failure. Drop anything you cannot substantiate.
|
||||
4. Synthesize a verdict: ship, ship after fixes, or do not ship. List the confirmed findings ordered by severity, each with its location and failure scenario, then any open questions. If nothing survived verification, say so plainly instead of inventing nitpicks.`,
|
||||
},
|
||||
...MEDIA_AGENT_SKILLS,
|
||||
];
|
||||
|
||||
export const BUILTIN_AGENT_SKILLS: Array<{
|
||||
|
||||
@ -8,6 +8,7 @@ import {
|
||||
USER_FACT_CATEGORIES,
|
||||
} from "@rakazo/contracts";
|
||||
import { z } from "zod";
|
||||
import { mediaAgentTools } from "./media-tools.js";
|
||||
|
||||
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
||||
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
||||
@ -963,6 +964,8 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
required: ["bot", "title", "brief"],
|
||||
},
|
||||
},
|
||||
// Offered only when Media Studio is configured (filterMediaTools in the executor).
|
||||
...mediaAgentTools,
|
||||
];
|
||||
|
||||
/** Agent-connection tools, exposed only when the messaging surface is enabled. */
|
||||
|
||||
@ -13,6 +13,7 @@ import type {
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { PrismaClient, ThreadEvents } from "@rakazo/db";
|
||||
import type { CloudAgentConnection } from "./cloud-agent-factory.js";
|
||||
import type { MediaServices } from "./media-context.js";
|
||||
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
|
||||
import type { RemoteTransportDependencies } from "./remote-mcp.js";
|
||||
import type { EncryptedSecretStore } from "./secrets.js";
|
||||
@ -45,6 +46,8 @@ export interface ExecutorDeps {
|
||||
secretHttp?: RemoteTransportDependencies;
|
||||
/** Remote cloud coding agents. Null/omit means tools stay uninjected. */
|
||||
cloudAgent?: CloudAgentConnection | null;
|
||||
/** Media Studio services, read per run; null or omitted keeps the media_* tools hidden. */
|
||||
media?: () => MediaServices | null;
|
||||
/** Aborted when createApp stop() begins so in-flight continueRun boot waits exit promptly. */
|
||||
shutdownSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
@ -290,6 +290,9 @@ import {
|
||||
needsOAuthProbe,
|
||||
parseMcpServerToolArgs,
|
||||
} from "./mcp-server-tool.js";
|
||||
import { filterMediaSkills } from "./media-skills.js";
|
||||
import { filterMediaTools, isMediaToolName } from "./media-tools.js";
|
||||
import { executeMediaTool } from "./media-tools-execute.js";
|
||||
import { loadAgentMemoryContext } from "./memory-context.js";
|
||||
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
|
||||
import { selectMemoryTools } from "./memory-tools.js";
|
||||
@ -1127,7 +1130,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
listAgentSkillRecords(deps.prisma, {
|
||||
spaceId: run.spaceId,
|
||||
userId: run.userId,
|
||||
}),
|
||||
}).then((records) => filterMediaSkills(records, Boolean(deps.media?.()))),
|
||||
deps.prisma.agentSecret.findMany({
|
||||
where: agentSecretScopeForRun(run.spaceId, run.userId),
|
||||
select: {
|
||||
@ -1477,18 +1480,22 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
select: { id: true },
|
||||
}))
|
||||
: false;
|
||||
const builtins = selectBuiltinToolsForRun({
|
||||
graphicalToolsAllowed,
|
||||
pageBrowserAllowed,
|
||||
groupId: thread.groupId,
|
||||
trigger: run.trigger,
|
||||
semanticMemoryEnabled,
|
||||
cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId),
|
||||
messagingChannelRun,
|
||||
agentConnectionsEnabled: hasMessagingIdentity,
|
||||
foreignAgentRun,
|
||||
bot,
|
||||
});
|
||||
const mediaServices = deps.media?.() ?? null;
|
||||
const builtins = filterMediaTools(
|
||||
selectBuiltinToolsForRun({
|
||||
graphicalToolsAllowed,
|
||||
pageBrowserAllowed,
|
||||
groupId: thread.groupId,
|
||||
trigger: run.trigger,
|
||||
semanticMemoryEnabled,
|
||||
cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId),
|
||||
messagingChannelRun,
|
||||
agentConnectionsEnabled: hasMessagingIdentity,
|
||||
foreignAgentRun,
|
||||
bot,
|
||||
}),
|
||||
mediaServices !== null,
|
||||
);
|
||||
const exposedConnectorTools = discovered.filter(
|
||||
(tool) => !builtinAgentTools.some((builtin) => builtin.name === tool.name),
|
||||
);
|
||||
@ -2635,6 +2642,16 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
);
|
||||
}
|
||||
|
||||
if (isMediaToolName(name)) {
|
||||
return finish(
|
||||
await executeMediaTool(
|
||||
mediaServices,
|
||||
{ spaceId: run.spaceId, userId: run.userId },
|
||||
name,
|
||||
args,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (name.startsWith("cloud_agent_")) {
|
||||
return finish(
|
||||
await executeCloudAgentTool(
|
||||
@ -3824,6 +3841,9 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
pluginLine,
|
||||
agentSkillsLine,
|
||||
taughtSkillsLine,
|
||||
exposedToolNames.has("media_search")
|
||||
? "For photo and video work, read the Video Producer skill with skill_read before using the media_* tools."
|
||||
: undefined,
|
||||
exposedToolNames.has("render_plot")
|
||||
? 'For charts and data visualization, use the render_plot tool: it renders bar, line, scatter, histogram, heatmap, faceted and many more chart types from a JSON spec and attaches the PNG to the chat. Call render_plot with {"help": true} before your first chart to read the full guide.'
|
||||
: undefined,
|
||||
|
||||
@ -118,6 +118,7 @@ export {
|
||||
} from "./media-render.js";
|
||||
export * from "./media-services.js";
|
||||
export * from "./media-sidecar.js";
|
||||
export * from "./media-skills.js";
|
||||
export * from "./media-zip.js";
|
||||
export * from "./memory-provider-factory.js";
|
||||
export * from "./messaging-context.js";
|
||||
|
||||
@ -18,6 +18,7 @@ import { MODEL_TIERS, ModelTierSchema } from "@rakazo/contracts";
|
||||
import { getLogger } from "@rakazo/logging";
|
||||
import { isToolPauseResult } from "./approval-effect.js";
|
||||
import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js";
|
||||
import { describeMediaToolActivity } from "./media-tools.js";
|
||||
import {
|
||||
normalizeOpenAiToolParameters,
|
||||
openAiToolParametersNeedNormalization,
|
||||
@ -474,6 +475,8 @@ export function describeToolActivity(toolName: string, args: unknown): string {
|
||||
if (toolName === "attach_file") return `Attaching ${detail(record.path)}`;
|
||||
if (toolName === "open_path") return `Opening ${detail(record.path)}`;
|
||||
if (toolName === "render_plot") return "Rendering a chart";
|
||||
const media = describeMediaToolActivity(toolName, record);
|
||||
if (media) return media;
|
||||
if (toolName === "add_mcp_server") return `Connecting MCP server: ${detail(record.name)}`;
|
||||
if (toolName === "computer_observe") return "Looking at the screen";
|
||||
if (toolName === "browser_navigate")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user