import type { AdapterContext, BrowserActRequest, BrowserActResult, BrowserProvider, BrowserSnapshotResult, ComputerRef, } from "@rakazo/adapter-kit"; import { ComputerPointerPayload } from "@rakazo/contracts"; import { DEFAULT_COMPUTER_SCREEN } from "@rakazo/core"; import { describe, expect, it } from "vitest"; import { browserActFromTool, browserSnapshotFromTool } from "./browser-tools.js"; import { BrowserPointerTargets, computerPointerPayload, computerPointerSteps, } from "./computer-pointer.js"; import { parseComputerActions } from "./computer-tools.js"; const context: AdapterContext = { operationId: "1", traceId: "1", spaceId: "w", userId: "u", signal: new AbortController().signal, }; const computer: ComputerRef = { id: "c1", botId: "b1", kind: "docker", providerRef: "c1" }; describe("computer pointer steps", () => { it("maps a computer_act batch to cursor steps in order", () => { const steps = computerPointerSteps( parseComputerActions([ { kind: "move", x: 10, y: 20 }, { kind: "click", x: 30, y: 40, double: true }, { kind: "click", x: 50, y: 60, button: "right" }, { kind: "type", text: "hello" }, { kind: "key", key: "Enter" }, { kind: "wait", ms: 100 }, { kind: "scroll", direction: "down" }, { kind: "down", x: 1, y: 2 }, { kind: "up", x: 3, y: 4 }, ]), ); expect(steps).toEqual([ { action: "move", x: 10, y: 20 }, { action: "double_click", x: 30, y: 40 }, { action: "right_click", x: 50, y: 60 }, { action: "type" }, { action: "key" }, { action: "scroll" }, { action: "down", x: 1, y: 2 }, { action: "up", x: 3, y: 4 }, ]); const payload = computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps); expect(ComputerPointerPayload.parse(payload)).toEqual(payload); }); it("keeps two separate clicks at different points apart", () => { expect( computerPointerSteps( parseComputerActions([ { kind: "click", x: 1, y: 1 }, { kind: "click", x: 2, y: 2 }, ]), ), ).toEqual([ { action: "click", x: 1, y: 1 }, { action: "click", x: 2, y: 2 }, ]); }); it("announces nothing for a batch without visible steps", () => { const steps = computerPointerSteps(parseComputerActions([{ kind: "wait", ms: 10 }])); expect(computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps)).toBeUndefined(); }); }); describe("browser pointer targets", () => { it("points at element centers and stops at the first unknown ref", () => { const targets = new BrowserPointerTargets(); targets.remember({ screen: { width: 1280, height: 800 }, elements: [ { ref: "e1", role: "textbox", name: "Name", box: [100, 200, 40, 20] }, { ref: "e2", role: "button", name: "Save", box: [300, 400, 11, 11] }, { ref: "e3", role: "button", name: "Hidden" }, // Untrusted helper output never becomes a cursor target. { ref: "e4", role: "button", name: "Bad", box: [-5, 0, 10, 10] as never }, ], }); expect( targets.payloadFor([ { kind: "fill", ref: "e1", text: "x" }, { kind: "hover", ref: "e2" }, { kind: "select", ref: "e2", text: "Save" }, { kind: "press", key: "Enter" }, { kind: "scroll", direction: "down" }, { kind: "wait", ms: 50 }, { kind: "click", ref: "e2" }, { kind: "click", ref: "e3" }, { kind: "click", ref: "e1" }, ]), ).toEqual({ kind: "pointer", screen: { width: 1280, height: 800 }, steps: [ { action: "type", x: 120, y: 210 }, { action: "move", x: 306, y: 406 }, { action: "click", x: 306, y: 406 }, { action: "key" }, { action: "scroll" }, { action: "click", x: 306, y: 406 }, ], }); expect(targets.payloadFor([{ kind: "click", ref: "e4" }])).toBeUndefined(); targets.forget(); expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined(); }); it("needs a reported screen size", () => { const targets = new BrowserPointerTargets(); targets.remember({ elements: [{ ref: "e1", role: "button", name: "A", box: [1, 1, 2, 2] }] }); expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined(); }); }); describe("page browser tools with a visible cursor", () => { it("announces before acting and keeps geometry away from the model", async () => { const order: string[] = []; const snapshot: BrowserSnapshotResult = { url: "https://example.test/", title: "Example", tree: '- button "Save" [e1]', screen: { width: 1280, height: 800 }, elements: [{ ref: "e1", role: "button", name: "Save", box: [10, 20, 30, 40] }], }; const browser = { snapshot: async () => snapshot, act: async (_computer: ComputerRef, request: BrowserActRequest) => { order.push(`act:${request.actions.length}`); return { ok: true, completed: 1, ...snapshot } satisfies BrowserActResult; }, } as unknown as BrowserProvider; const announced: unknown[] = []; const pointer = { targets: new BrowserPointerTargets(), announce: async (payload: ComputerPointerPayload) => { order.push("announce"); announced.push(payload); }, }; const seen = await browserSnapshotFromTool(browser, computer, context, {}, pointer); expect(JSON.stringify(seen)).not.toMatch(/box|screen/); const acted = await browserActFromTool( browser, computer, context, { actions: [{ kind: "click", ref: "e1" }] }, pointer, ); expect(order).toEqual(["announce", "act:1"]); expect(announced).toEqual([ { kind: "pointer", screen: { width: 1280, height: 800 }, steps: [{ action: "click", x: 25, y: 40 }], }, ]); expect(JSON.stringify(acted)).not.toMatch(/box|screen/); }); });