import { randomUUID } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { ComposioEmulator } from "@rakazo/adapters"; import { describe, expect, it } from "vitest"; import { sessionCookieHeader } from "./index.js"; import { startModelEmulator } from "./model-emulator.js"; type App = { request: (input: string, init?: RequestInit) => Promise }; const databaseAvailable = process.env.VERIFY_DATABASE === "1" && Boolean(process.env.DATABASE_URL); const fixtureOrigin = "http://127.0.0.1:5173"; describe.skipIf(!databaseAvailable)("offline Pi product journey", () => { it("uses a saved model connection and the real executor to persist a file and completed run", async () => { const fixtureKey = "offline-product-fixture-key"; const model = await startModelEmulator({ apiKey: fixtureKey, steps: [ { expect(request) { const system = request.messages.find( (message) => message.role === "system" || message.role === "developer", ); const instructions = String(system?.content ?? ""); expect(instructions.startsWith("You are File fixture.")).toBe(true); expect(instructions).toContain("Complete the task."); expect(instructions).toContain("\nName: Offline fixture\n"); expect(JSON.stringify(request.messages)).toContain("Save hello to notes/result.txt."); expect(request.tools).toContainEqual( expect.objectContaining({ function: expect.objectContaining({ name: "write_file" }), }), ); }, response: { type: "tool", id: "product-write", name: "write_file", arguments: { path: "notes/result.txt", content: "hello" }, }, }, { expect(request) { const result = request.messages.findLast((message) => message.role === "tool"); expect(result?.tool_call_id).toBe("product-write"); expect(JSON.parse(String(result?.content))).toMatchObject({ ok: true, path: "notes/result.txt", }); }, response: { type: "text", text: "Saved notes/result.txt." }, }, ], }); const dataDir = await mkdtemp(path.join(tmpdir(), "rakazo-offline-product-")); let stop: (() => Promise) | undefined; try { const { createApp } = await import("../../../apps/api/src/app.ts"); const handles = await createApp({ databaseUrl: process.env.DATABASE_URL!, realtimeDatabaseUrl: process.env.DATABASE_URL!, authUrl: fixtureOrigin, webOrigin: fixtureOrigin, dataDir, sandboxProvider: "fake", agentRuntime: "pi", wakeupDriver: "memory", signupsEnabled: "true", composio: new ComposioEmulator(), encryptionKey: "offline-model-fixture-encryption-key", }); stop = handles.stop; const signup = await handles.app.request("/api/auth/sign-up/email", { method: "POST", headers: { "content-type": "application/json", origin: fixtureOrigin }, body: JSON.stringify({ email: `offline-pi-${randomUUID()}@rakazo.test`, password: "password12", name: "Offline fixture", }), }); expect(signup.status).toBeLessThan(400); const cookie = sessionCookieHeader(signup); await rpc(handles.app, cookie, "models/connect", { provider: model.model.provider, modelId: model.model.id, baseUrl: model.baseUrl, apiKey: fixtureKey, }); const bot = await rpc<{ id: string }>(handles.app, cookie, "bots/create", { name: "File fixture", title: "", description: "", instructions: "Complete the task.", notifyOnFinish: false, }); await rpc(handles.app, cookie, "bots/update", { botId: bot.id, modelProvider: model.model.provider, modelId: model.model.id, }); const sent = await rpc<{ runId: string }>(handles.app, cookie, "threads/send", { botId: bot.id, text: "Save hello to notes/result.txt.", }); await expect .poll( async () => { const run = await handles.prisma.run.findUnique({ where: { id: sent.runId }, select: { status: true }, }); if (run?.status === "failed") model.assertComplete(); return run?.status; }, { timeout: 15_000, interval: 100 }, ) .toBe("completed"); model.assertComplete(); const file = await rpc<{ content: string }>(handles.app, cookie, "computer/readFile", { botId: bot.id, path: "notes/result.txt", }); expect(file.content).toBe("hello"); const thread = await rpc<{ messages: Array<{ role: string; blocks: unknown[] }> }>( handles.app, cookie, "threads/get", { botId: bot.id }, ); const messages = JSON.stringify(thread.messages); expect(messages).toContain("Saved notes/result.txt."); expect(messages).not.toContain(fixtureKey); const tools = await handles.prisma.event.findMany({ where: { botId: bot.id, type: "agent.tool.called" }, select: { payload: true }, }); expect(tools).toHaveLength(1); expect(JSON.stringify(tools[0])).toContain("write_file"); } finally { try { await stop?.(); } finally { await model.close(); await rm(dataDir, { recursive: true, force: true }); } } }, 30_000); it("runs a bot's tier and a helper's tier on models of one router connection", async () => { const fixtureKey = "offline-router-fixture-key"; const lastToolResult = (request: { messages: Array<{ role: string; content?: unknown }> }) => String(request.messages.findLast((message) => message.role === "tool")?.content); const model = await startModelEmulator({ apiKey: fixtureKey, modelId: "router-standard", routedModelIds: ["router-fast", "router-code"], steps: [ { expect(request) { expect(request.model).toBe("router-code"); const helper = request.tools?.find((tool) => tool.function.name === "run_subagent"); expect(JSON.stringify(helper?.function.parameters)).toContain("model_tier"); }, response: { type: "tool", id: "tier-helper", name: "run_subagent", arguments: { name: "scout", task: "Count to three.", model_tier: "fast" }, }, }, { expect(request) { expect(request.model).toBe("router-fast"); expect(JSON.stringify(request.messages)).toContain("Count to three."); }, response: { type: "text", text: "one two three" }, }, { expect(request) { expect(request.model).toBe("router-code"); expect(lastToolResult(request)).toContain("one two three"); }, response: { type: "text", text: "The helper counted." }, }, { // With its tier cleared the bot is back on the connection's saved model. expect(request) { expect(request.model).toBe("router-standard"); }, response: { type: "text", text: "Back on the saved model." }, }, ], }); const dataDir = await mkdtemp(path.join(tmpdir(), "rakazo-offline-tiers-")); let stop: (() => Promise) | undefined; try { const { createApp } = await import("../../../apps/api/src/app.ts"); const handles = await createApp({ databaseUrl: process.env.DATABASE_URL!, realtimeDatabaseUrl: process.env.DATABASE_URL!, authUrl: fixtureOrigin, webOrigin: fixtureOrigin, dataDir, sandboxProvider: "fake", agentRuntime: "pi", wakeupDriver: "memory", signupsEnabled: "true", composio: new ComposioEmulator(), encryptionKey: "offline-model-fixture-encryption-key", }); stop = handles.stop; const signup = await handles.app.request("/api/auth/sign-up/email", { method: "POST", headers: { "content-type": "application/json", origin: fixtureOrigin }, body: JSON.stringify({ email: `offline-tiers-${randomUUID()}@rakazo.test`, password: "password12", name: "Offline tiers", }), }); expect(signup.status).toBeLessThan(400); const cookie = sessionCookieHeader(signup); const router = await rpc<{ id: string }>(handles.app, cookie, "models/connect", { provider: model.model.provider, modelId: model.model.id, baseUrl: model.baseUrl, apiKey: fixtureKey, }); for (const [tier, modelId] of [ ["code", "router-code"], ["fast", "router-fast"], ]) { await rpc(handles.app, cookie, "models/tiers/set", { tier, model: { credentialId: router.id, modelId }, }); } const bot = await rpc<{ id: string }>(handles.app, cookie, "bots/create", { name: "Tier fixture", title: "", description: "", instructions: "Complete the task.", notifyOnFinish: false, }); await rpc(handles.app, cookie, "bots/update", { botId: bot.id, modelTier: "code" }); const completedRun = async (text: string) => { const sent = await rpc<{ runId: string }>(handles.app, cookie, "threads/send", { botId: bot.id, text, }); await expect .poll( async () => { const run = await handles.prisma.run.findUnique({ where: { id: sent.runId }, select: { status: true }, }); if (run?.status === "failed") model.assertComplete(); return run?.status; }, { timeout: 15_000, interval: 100 }, ) .toBe("completed"); return handles.prisma.run.findUniqueOrThrow({ where: { id: sent.runId }, select: { modelProvider: true, modelId: true }, }); }; await expect(completedRun("Ask a fast helper to count.")).resolves.toEqual({ modelProvider: "openai-compatible", modelId: "router-code", }); await rpc(handles.app, cookie, "models/tiers/set", { tier: "code", model: null }); await expect(completedRun("Answer directly.")).resolves.toEqual({ modelProvider: "openai-compatible", modelId: "router-standard", }); model.assertComplete(); const thread = await rpc<{ messages: unknown[] }>(handles.app, cookie, "threads/get", { botId: bot.id, }); const messages = JSON.stringify(thread.messages); expect(messages).toContain("The helper counted."); expect(messages).toContain("Back on the saved model."); expect(messages).not.toContain(fixtureKey); } finally { try { await stop?.(); } finally { await model.close(); await rm(dataDir, { recursive: true, force: true }); } } }, 45_000); }); async function rpc( app: App, cookie: string, procedure: string, input: unknown = {}, ): Promise { const response = await app.request(`/rpc/${procedure}`, { method: "POST", headers: { "content-type": "application/json", cookie, origin: fixtureOrigin }, body: JSON.stringify({ json: input }), }); const body = (await response.json()) as { json?: T; error?: { message?: string } }; if (response.status >= 400 || body.error) throw new Error(`${procedure}: ${body.error?.message ?? response.status}`); return body.json as T; }