import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { ComposioEmulator, diagnoseReleaseWatchRun, githubToolResultHasSeededRelease, RELEASE_WATCH_GITHUB_TOOL_NAMES, resolveReleaseWatchEvalModelId, } from "@rakazo/adapters"; import { loadRootEnv } from "@rakazo/core/node/load-root-env"; import { afterAll, describe, expect, it } from "vitest"; import { sessionCookieHeader } from "./index.js"; if (process.env.VERIFY_PROVIDERS) loadRootEnv(); const live = Boolean( process.env.VERIFY_PROVIDERS && process.env.OPENROUTER_API_KEY && process.env.DATABASE_URL, ); const describeLive = live ? describe : describe.skip; describeLive("live release-watch eval (GPT 5.6 Luna + GitHub emulator)", () => { let stop: (() => Promise) | undefined; let dataDir: string | undefined; const composio = new ComposioEmulator(); afterAll(async () => { try { await stop?.(); } finally { if (dataDir) rmSync(dataDir, { recursive: true, force: true }); } }); it("writes a concrete routine and reads seeded releases via GitHub tools, not the browser", async () => { const { modelId, label } = resolveReleaseWatchEvalModelId(process.env); process.env.PI_DEFAULT_MODEL = modelId; const { createApp } = await import("../../../apps/api/src/app.ts"); dataDir = mkdtempSync(path.join(tmpdir(), "rakazo-release-watch-")); const handles = await createApp({ databaseUrl: process.env.DATABASE_URL!, dataDir, sandboxProvider: "fake", agentRuntime: "pi", composio, }); stop = handles.stop; const stamp = Date.now(); const signup = await handles.app.request("/api/auth/sign-up/email", { method: "POST", headers: { "content-type": "application/json", origin: "http://127.0.0.1:5173" }, body: JSON.stringify({ email: `release-watch-${stamp}@rakazo.test`, password: "password12", name: "Release Watch", }), }); expect(signup.status).toBeLessThan(400); const cookie = sessionCookieHeader(signup); const me = await rpc<{ spaceId: string; userId: string }>(handles.app, cookie, "me"); const started = await rpc<{ connectionId: string; authorizationUrl: null }>( handles.app, cookie, "connections/begin", { connectorId: "composio", provider: "GITHUB", displayName: "GitHub" }, ); expect(started.authorizationUrl).toBeNull(); const discoveredGithubTools = await composio.discoverTools({ operationId: "release-watch-canary", traceId: "release-watch-canary", spaceId: me.spaceId, userId: me.userId, signal: new AbortController().signal, connectedConnections: [ { id: started.connectionId, connectorId: "composio", externalId: "GITHUB", displayName: "GitHub", }, ], }); expect(discoveredGithubTools.map((tool) => tool.name)).toEqual( expect.arrayContaining([...RELEASE_WATCH_GITHUB_TOOL_NAMES]), ); const bot = await rpc<{ id: string }>(handles.app, cookie, "bots/create", { name: "Chief", title: "Chief of staff", description: "Release watch eval", instructions: "You are a chief of staff.", notifyOnFinish: true, }); // Mirror the Discord user ask: no tool names, no Bing/browser coaching. // schedule_create + connected-plugin guidance should steer the model. await rpc(handles.app, cookie, "threads/send", { botId: bot.id, text: [ "Create a daily task to watch for new releases of elie222/rakazo", "and stay current on the project's capabilities.", ].join(" "), }); await waitFor(handles.app, cookie, bot.id, 180_000); const routines = await rpc>( handles.app, cookie, "routines/list", { botId: bot.id }, ); expect(routines.length).toBeGreaterThan(0); const routine = routines.find((row) => /rakazo|release/i.test(`${row.name}\n${row.prompt}`)) ?? routines[0]!; const dueAt = new Date(Date.now() - 1_000); await handles.prisma.routine.update({ where: { id: routine.id }, data: { nextRunAt: dueAt, active: true }, }); composio.executions.length = 0; const priorToolEvent = await handles.prisma.event.findFirst({ where: { botId: bot.id, type: "agent.tool.called" }, orderBy: { seq: "desc" }, select: { seq: true }, }); const routineToolSeqFloor = priorToolEvent?.seq ?? -1; await handles.jobs.enqueue({ name: "routine.wakeup", payload: { routineId: routine.id, scheduledFor: dueAt.toISOString() }, }); const snap = await waitFor(handles.app, cookie, bot.id, 180_000, { requireSeenRun: true }); const routineRunId = snap.run?.id; const toolEvents = await handles.prisma.event.findMany({ where: { botId: bot.id, type: "agent.tool.called", seq: { gt: routineToolSeqFloor }, ...(routineRunId ? { runId: routineRunId } : {}), }, orderBy: { seq: "asc" }, }); const calledFromEvents = toolEvents.map((event) => { const payload = event.payload as { name?: string }; return String(payload.name ?? ""); }); const routineExecutions = composio.executions.filter( (row) => row.botId === bot.id && (routineRunId ? row.runId === routineRunId : true), ); const calledToolNames = [...calledFromEvents, ...routineExecutions.map((row) => row.tool)]; const seededReleaseTags = composio.listGithubReleases().map((release) => release.tag); const resultText = JSON.stringify(snap); const githubToolResults = routineExecutions .filter((row) => (RELEASE_WATCH_GITHUB_TOOL_NAMES as readonly string[]).includes(row.tool)) .map((row) => row.result); const diagnosis = diagnoseReleaseWatchRun({ // Use tools the bot actually discovered — do not hardcode availability. availableToolNames: discoveredGithubTools.map((tool) => tool.name), calledToolNames, routinePrompt: routine.prompt, resultText, seededReleaseTags, githubToolResults, }); if (!diagnosis.pass) { throw new Error( [ "release-watch eval failed", `model=${label} (${modelId})`, `routinePrompt=${JSON.stringify(routine.prompt)}`, `calledTools=${JSON.stringify(calledToolNames)}`, `discoveredGithubTools=${JSON.stringify(discoveredGithubTools.map((tool) => tool.name))}`, diagnosis.summary, ].join("\n"), ); } expect( githubToolResults.some((result) => githubToolResultHasSeededRelease(result, seededReleaseTags), ), ).toBe(true); }, 420_000); }); type App = { request: (input: string, init?: RequestInit) => Promise }; type Snap = { messages: Array<{ role: string; blocks: unknown[] }>; run: { id: string; status: string; botId?: string; trigger?: string; routineId?: string | null; } | null; }; async function rpc(app: App, cookie: string, proc: string, body: unknown = {}): Promise { const res = await app.request(`/rpc/${proc}`, { method: "POST", headers: { "content-type": "application/json", cookie, origin: "http://127.0.0.1:5173" }, body: JSON.stringify({ json: body }), }); const parsed = (await res.json()) as { json?: T; error?: { message?: string } }; if (res.status >= 400 || parsed.error) { throw new Error(`${proc} ${res.status}: ${parsed.error?.message ?? "failed"}`); } return parsed.json as T; } async function waitFor( app: App, cookie: string, botId: string, ms: number, options?: { requireSeenRun?: boolean }, ): Promise { const start = Date.now(); let last: Snap | null = null; let seenRun = false; let lastSeenRun: Snap["run"] = null; while (Date.now() - start < ms) { last = await rpc(app, cookie, "threads/get", { botId }); if (last.run) { seenRun = true; lastSeenRun = last.run; } const terminal = !!last.run && ["completed", "failed", "cancelled"].includes(last.run.status); // Keep lastSeenRun for tool attribution if the active run slot clears after completion. if (terminal) return { ...last, run: last.run ?? lastSeenRun }; if (!last.run && (!options?.requireSeenRun || seenRun)) { return { ...last, run: lastSeenRun }; } await new Promise((r) => setTimeout(r, 400)); } throw new Error(`timeout waiting for live model turn: ${JSON.stringify(last)}`); }