evOo-local/packages/adapters/src/computer-pointer.test.ts

import type {
  AdapterContext,
  BrowserActRequest,
  BrowserActResult,
  BrowserProvider,
  BrowserSnapshotResult,
  ComputerRef,
} from "@rakazo/adapter-kit";
import { ComputerPointerPayload } from "@rakazo/contracts";
import { DEFAULT_COMPUTER_SCREEN } from "@rakazo/core";
import { describe, expect, it } from "vitest";
import { browserActFromTool, browserSnapshotFromTool } from "./browser-tools.js";
import {
  BrowserPointerTargets,
  computerPointerPayload,
  computerPointerSteps,
} from "./computer-pointer.js";
import { parseComputerActions } from "./computer-tools.js";

const context: AdapterContext = {
  operationId: "1",
  traceId: "1",
  spaceId: "w",
  userId: "u",
  signal: new AbortController().signal,
};
const computer: ComputerRef = { id: "c1", botId: "b1", kind: "docker", providerRef: "c1" };

describe("computer pointer steps", () => {
  it("maps a computer_act batch to cursor steps in order", () => {
    const steps = computerPointerSteps(
      parseComputerActions([
        { kind: "move", x: 10, y: 20 },
        { kind: "click", x: 30, y: 40, double: true },
        { kind: "click", x: 50, y: 60, button: "right" },
        { kind: "type", text: "hello" },
        { kind: "key", key: "Enter" },
        { kind: "wait", ms: 100 },
        { kind: "scroll", direction: "down" },
        { kind: "down", x: 1, y: 2 },
        { kind: "up", x: 3, y: 4 },
      ]),
    );
    expect(steps).toEqual([
      { action: "move", x: 10, y: 20 },
      { action: "double_click", x: 30, y: 40 },
      { action: "right_click", x: 50, y: 60 },
      { action: "type" },
      { action: "key" },
      { action: "scroll" },
      { action: "down", x: 1, y: 2 },
      { action: "up", x: 3, y: 4 },
    ]);
    const payload = computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps);
    expect(ComputerPointerPayload.parse(payload)).toEqual(payload);
  });

  it("keeps two separate clicks at different points apart", () => {
    expect(
      computerPointerSteps(
        parseComputerActions([
          { kind: "click", x: 1, y: 1 },
          { kind: "click", x: 2, y: 2 },
        ]),
      ),
    ).toEqual([
      { action: "click", x: 1, y: 1 },
      { action: "click", x: 2, y: 2 },
    ]);
  });

  it("announces nothing for a batch without visible steps", () => {
    const steps = computerPointerSteps(parseComputerActions([{ kind: "wait", ms: 10 }]));
    expect(computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps)).toBeUndefined();
  });
});

describe("browser pointer targets", () => {
  it("points at element centers and stops at the first unknown ref", () => {
    const targets = new BrowserPointerTargets();
    targets.remember({
      screen: { width: 1280, height: 800 },
      elements: [
        { ref: "e1", role: "textbox", name: "Name", box: [100, 200, 40, 20] },
        { ref: "e2", role: "button", name: "Save", box: [300, 400, 11, 11] },
        { ref: "e3", role: "button", name: "Hidden" },
        // Untrusted helper output never becomes a cursor target.
        { ref: "e4", role: "button", name: "Bad", box: [-5, 0, 10, 10] as never },
      ],
    });
    expect(
      targets.payloadFor([
        { kind: "fill", ref: "e1", text: "x" },
        { kind: "click", ref: "e2" },
        { kind: "click", ref: "e3" },
        { kind: "click", ref: "e1" },
      ]),
    ).toEqual({
      kind: "pointer",
      screen: { width: 1280, height: 800 },
      steps: [
        { action: "type", x: 120, y: 210 },
        { action: "click", x: 306, y: 406 },
      ],
    });
    expect(targets.payloadFor([{ kind: "click", ref: "e4" }])).toBeUndefined();
    targets.forget();
    expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
  });

  it("needs a reported screen size", () => {
    const targets = new BrowserPointerTargets();
    targets.remember({ elements: [{ ref: "e1", role: "button", name: "A", box: [1, 1, 2, 2] }] });
    expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
  });
});

describe("page browser tools with a visible cursor", () => {
  it("announces before acting and keeps geometry away from the model", async () => {
    const order: string[] = [];
    const snapshot: BrowserSnapshotResult = {
      url: "https://example.test/",
      title: "Example",
      tree: '- button "Save" [e1]',
      screen: { width: 1280, height: 800 },
      elements: [{ ref: "e1", role: "button", name: "Save", box: [10, 20, 30, 40] }],
    };
    const browser = {
      snapshot: async () => snapshot,
      act: async (_computer: ComputerRef, request: BrowserActRequest) => {
        order.push(`act:${request.actions.length}`);
        return { ok: true, completed: 1, ...snapshot } satisfies BrowserActResult;
      },
    } as unknown as BrowserProvider;
    const announced: unknown[] = [];
    const pointer = {
      targets: new BrowserPointerTargets(),
      announce: async (payload: ComputerPointerPayload) => {
        order.push("announce");
        announced.push(payload);
      },
    };

    const seen = await browserSnapshotFromTool(browser, computer, context, {}, pointer);
    expect(JSON.stringify(seen)).not.toMatch(/box|screen/);

    const acted = await browserActFromTool(
      browser,
      computer,
      context,
      { actions: [{ kind: "click", ref: "e1" }] },
      pointer,
    );
    expect(order).toEqual(["announce", "act:1"]);
    expect(announced).toEqual([
      {
        kind: "pointer",
        screen: { width: 1280, height: 800 },
        steps: [{ action: "click", x: 25, y: 40 }],
      },
    ]);
    expect(JSON.stringify(acted)).not.toMatch(/box|screen/);
  });
});