evOo-local/packages/testkit/src/computer-approval.postgres.test.ts

import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { ComposioEmulator, FakeSandboxProvider } from "@rakazo/adapters";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { sessionCookieHeader } from "./index.js";
import { type ModelEmulatorStep, startModelEmulator } from "./model-emulator.js";

type App = { request: (input: string, init?: RequestInit) => Promise<Response> };
const databaseAvailable = process.env.VERIFY_DATABASE === "1" && Boolean(process.env.DATABASE_URL);
const fixtureOrigin = "http://127.0.0.1:5173";

describe.skipIf(!databaseAvailable)("offline Pi computer approval", () => {
  beforeAll(() => {
    // Use the existing compatible-endpoint capability declaration so the real
    // executor exposes computer tools without mocking its model vision gate.
    vi.stubEnv("RAKAZO_OPENAI_COMPATIBLE_VISION_MODELS", "offline-fixture");
  });
  afterAll(() => vi.unstubAllEnvs());

  it.each(["allow", "deny"] as const)(
    "%s enforces the persisted computer action before any sandbox effect",
    async (answer) => {
      let sandbox: FakeSandboxProvider;
      const approvedArgs = {
        actions: [{ kind: "type", text: "approved clipboard content" }],
        observe: false,
        settle_ms: 0,
      };
      const resultStep =
        (id: string): ModelEmulatorStep["expect"] =>
        (request) => {
          const result = request.messages.findLast((message) => message.role === "tool");
          expect(result?.tool_call_id).toBe(id);
          const body = JSON.parse(String(result?.content));
          if (answer === "allow") expect(body).toMatchObject({ ok: true, completed: 1 });
          else expect(JSON.stringify(body)).toMatch(/denied/i);
          // Inspect the provider state while the run still owns its screen;
          // completed runs release that screen and discard its placeholder state.
          const screens = [...sandbox.boxes.values()].flatMap((box) => [...box.screens.values()]);
          expect(screens).not.toContain("unapproved replacement");
          if (answer === "allow") expect(screens).toContain("approved clipboard content");
          else expect(screens).not.toContain("approved clipboard content");
        };
      const fixtureKey = "offline-computer-approval-key";
      const model = await startModelEmulator({
        apiKey: fixtureKey,
        steps: [
          {
            expect(request) {
              expect(request.tools).toContainEqual(
                expect.objectContaining({
                  function: expect.objectContaining({ name: "computer_act" }),
                }),
              );
            },
            response: {
              type: "tool",
              id: "pending-action",
              name: "computer_act",
              arguments: approvedArgs,
            },
          },
          {
            expect(request) {
              expect(JSON.stringify(request.messages)).toContain("approved clipboard content");
            },
            response: {
              type: "tool",
              id: "resumed-action",
              name: "computer_act",
              // The executor must restore exactly what the user approved, even if
              // the model reconstructs different arguments after its run resumes.
              arguments:
                answer === "allow"
                  ? {
                      ...approvedArgs,
                      actions: [{ kind: "type", text: "unapproved replacement" }],
                    }
                  : approvedArgs,
            },
          },
          {
            // A later identical computer_act in this same resume is a new occurrence,
            // not a replay. This fixture only checks the approved request is restored.
            expect: resultStep("resumed-action"),
            response: { type: "text", text: "Finished the approval fixture." },
          },
        ],
      });
      const dataDir = await mkdtemp(path.join(tmpdir(), "rakazo-computer-approval-"));
      let stop: (() => Promise<void>) | undefined;
      try {
        const { createApp } = await import("../../../apps/api/src/app.ts");
        const handles = await createApp({
          databaseUrl: process.env.DATABASE_URL!,
          realtimeDatabaseUrl: process.env.DATABASE_URL!,
          authUrl: fixtureOrigin,
          webOrigin: fixtureOrigin,
          dataDir,
          sandboxProvider: "fake",
          agentRuntime: "pi",
          wakeupDriver: "memory",
          signupsEnabled: "true",
          composio: new ComposioEmulator(),
          encryptionKey: "offline-computer-fixture-encryption-key",
        });
        stop = handles.stop;
        expect(handles.sandbox).toBeInstanceOf(FakeSandboxProvider);
        sandbox = handles.sandbox as FakeSandboxProvider;
        // Keep the real fake-provider implementation; observe calls and its state
        // independently of the model's claims and the persisted effect record.
        const act = vi.spyOn(sandbox, "act");
        const signup = await handles.app.request("/api/auth/sign-up/email", {
          method: "POST",
          headers: { "content-type": "application/json", origin: fixtureOrigin },
          body: JSON.stringify({
            email: `computer-approval-${randomUUID()}@rakazo.test`,
            password: "password12",
            name: "Computer approval fixture",
          }),
        });
        expect(signup.status).toBeLessThan(400);
        const cookie = sessionCookieHeader(signup);
        await rpc(handles.app, cookie, "models/connect", {
          provider: model.model.provider,
          modelId: model.model.id,
          baseUrl: model.baseUrl,
          apiKey: fixtureKey,
        });
        const bot = await rpc<{ id: string }>(handles.app, cookie, "bots/create", {
          name: "Computer fixture",
          title: "",
          description: "",
          instructions: "Complete the computer task.",
          notifyOnFinish: false,
        });
        await rpc(handles.app, cookie, "bots/update", {
          botId: bot.id,
          modelProvider: model.model.provider,
          modelId: model.model.id,
        });
        const storedBot = await handles.prisma.bot.findUniqueOrThrow({ where: { id: bot.id } });
        await handles.prisma.actionApprovalRule.create({
          data: {
            spaceId: storedBot.spaceId,
            createdByUserId: storedBot.userId,
            effect: "require_approval",
            matchKind: "tool",
            matchValue: "computer_act",
          },
        });
        const sent = await rpc<{ runId: string }>(handles.app, cookie, "threads/send", {
          botId: bot.id,
          text: "Put approved clipboard content on the computer clipboard.",
        });
        const waitForRun = async (status: string) => {
          await expect
            .poll(
              async () => {
                const run = await handles.prisma.run.findUniqueOrThrow({
                  where: { id: sent.runId },
                });
                if (run.status === "failed") model.assertComplete();
                return run.status;
              },
              { timeout: 15_000, interval: 100 },
            )
            .toBe(status);
        };
        await waitForRun("waiting_input");
        expect(model.requests).toHaveLength(1);
        expect(act).not.toHaveBeenCalled();
        const effects = await handles.prisma.externalEffect.findMany({
          where: { runId: sent.runId },
        });
        expect(effects).toHaveLength(1);
        expect(effects[0]).toMatchObject({
          kind: "computer_act",
          status: "intended",
          request: approvedArgs,
        });
        const card = await handles.prisma.message.findFirstOrThrow({
          where: { runId: sent.runId, role: "bot" },
          orderBy: { seq: "desc" },
        });
        expect(card.blocks).toEqual(
          expect.arrayContaining([
            expect.objectContaining({ kind: "ask", approvalEffectId: effects[0]!.id }),
          ]),
        );
        await rpc(handles.app, cookie, "threads/answer", {
          botId: bot.id,
          runId: sent.runId,
          messageId: card.id,
          answer,
        });
        await waitForRun("completed");
        model.assertComplete();
        expect(act).toHaveBeenCalledTimes(answer === "allow" ? 1 : 0);
        if (answer === "allow") {
          expect(act.mock.calls[0]![1]).toEqual({
            actions: [{ kind: "clipboard", text: "approved clipboard content" }],
            observe: false,
            settleMs: 0,
          });
        }
        const settledEffects = await handles.prisma.externalEffect.findMany({
          where: { runId: sent.runId },
        });
        expect(settledEffects).toHaveLength(1);
        expect(settledEffects[0]).toMatchObject({
          id: effects[0]!.id,
          status: answer === "allow" ? "completed" : "denied",
          request: approvedArgs,
        });
      } finally {
        try {
          await stop?.();
        } finally {
          await model.close();
          await rm(dataDir, { recursive: true, force: true });
        }
      }
    },
    45_000,
  );
});

async function rpc<T>(
  app: App,
  cookie: string,
  procedure: string,
  input: unknown = {},
): Promise<T> {
  const response = await app.request(`/rpc/${procedure}`, {
    method: "POST",
    headers: { "content-type": "application/json", cookie, origin: fixtureOrigin },
    body: JSON.stringify({ json: input }),
  });
  const body = (await response.json()) as { json?: T; error?: { message?: string } };
  if (response.status >= 400 || body.error)
    throw new Error(`${procedure}: ${body.error?.message ?? response.status}`);
  return body.json as T;
}