import type { BrowserActStep, BrowserScreenBox, BrowserSnapshotNode, ComputerAction, } from "@rakazo/adapter-kit"; import { type ComputerPointerPayload, type ComputerPointerStep, MAX_COMPUTER_POINTER_STEPS, } from "@rakazo/contracts"; export type ComputerScreenSize = ComputerPointerPayload["screen"]; /** Pointer steps for a validated computer_act batch, in execution order. */ export function computerPointerSteps(actions: ComputerAction[]): ComputerPointerStep[] { const steps: ComputerPointerStep[] = []; for (let index = 0; index < actions.length; index += 1) { const action = actions[index]!; if (action.kind === "pointer") { const next = actions[index + 1]; // parseComputerActions expands a double click into two identical clicks. if ( action.type === "click" && next?.kind === "pointer" && next.type === "click" && next.x === action.x && next.y === action.y && next.button === action.button ) { steps.push({ action: "double_click", x: action.x, y: action.y }); index += 1; continue; } const kind = action.type === "click" ? action.button === "right" ? "right_click" : "click" : action.type; steps.push({ action: kind, x: action.x, y: action.y }); } else if (action.kind === "clipboard") { steps.push({ action: "type" }); } else if (action.kind === "key" || action.kind === "scroll") { steps.push({ action: action.kind }); } } return steps.slice(0, MAX_COMPUTER_POINTER_STEPS); } export function computerPointerPayload( screen: ComputerScreenSize, steps: ComputerPointerStep[], ): ComputerPointerPayload | undefined { if (steps.length === 0) return undefined; return { kind: "pointer", screen, steps: steps.slice(0, MAX_COMPUTER_POINTER_STEPS) }; } /** * Remembers where the page browser last saw each element on screen, so a browser_act batch * can be announced as pointer steps before it runs. Geometry never reaches the model. */ export class BrowserPointerTargets { private boxes = new Map(); private screen: ComputerScreenSize | undefined; remember(result: { elements?: BrowserSnapshotNode[]; screen?: ComputerScreenSize }) { if (!Array.isArray(result.elements)) return; this.boxes = new Map(); this.screen = validScreen(result.screen); for (const element of result.elements) { if (validBox(element.box)) this.boxes.set(element.ref, element.box); } } forget() { this.boxes = new Map(); this.screen = undefined; } payloadFor(actions: BrowserActStep[]): ComputerPointerPayload | undefined { if (!this.screen) return undefined; const steps: ComputerPointerStep[] = []; for (const action of actions) { const box = this.boxes.get(action.ref); // Later steps may target elements an earlier step reveals; stop at the first unknown one. if (!box) break; const [x, y, width, height] = box; steps.push({ action: action.kind === "click" ? "click" : "type", x: Math.round(x + width / 2), y: Math.round(y + height / 2), }); } return computerPointerPayload(this.screen, steps); } } /** Strips screen geometry from page-browser results before they reach the model. */ export function withoutBrowserGeometry( result: T, ): Omit { const { screen: _screen, ...rest } = result as T & { screen?: unknown }; if (!Array.isArray(rest.elements)) return rest; return { ...rest, elements: rest.elements.map(({ box: _box, ...element }) => element), }; } function validScreen(value: unknown): ComputerScreenSize | undefined { if (!value || typeof value !== "object") return undefined; const { width, height } = value as Record; if (!isPixel(width) || !isPixel(height) || width === 0 || height === 0) return undefined; return { width, height }; } function validBox(value: unknown): value is BrowserScreenBox { return ( Array.isArray(value) && value.length === 4 && value.every(isPixel) && (value[2] as number) > 0 && (value[3] as number) > 0 ); } function isPixel(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100_000; }