evOo-local/packages/adapters/src/computer-pointer.ts

import type {
  BrowserActStep,
  BrowserScreenBox,
  BrowserSnapshotNode,
  ComputerAction,
} from "@rakazo/adapter-kit";
import {
  type ComputerPointerPayload,
  type ComputerPointerStep,
  MAX_COMPUTER_POINTER_STEPS,
} from "@rakazo/contracts";
import { DEFAULT_COMPUTER_SCREEN } from "@rakazo/core";

export type ComputerScreenSize = ComputerPointerPayload["screen"];

/** Pointer steps for a validated computer_act batch, in execution order. */
export function computerPointerSteps(actions: ComputerAction[]): ComputerPointerStep[] {
  const steps: ComputerPointerStep[] = [];
  for (let index = 0; index < actions.length; index += 1) {
    const action = actions[index]!;
    if (action.kind === "pointer") {
      const next = actions[index + 1];
      // parseComputerActions expands a double click into two identical clicks.
      if (
        action.type === "click" &&
        next?.kind === "pointer" &&
        next.type === "click" &&
        next.x === action.x &&
        next.y === action.y &&
        next.button === action.button
      ) {
        steps.push({ action: "double_click", x: action.x, y: action.y });
        index += 1;
        continue;
      }
      const kind =
        action.type === "click"
          ? action.button === "right"
            ? "right_click"
            : "click"
          : action.type;
      steps.push({ action: kind, x: action.x, y: action.y });
    } else if (action.kind === "clipboard") {
      steps.push({ action: "type" });
    } else if (action.kind === "key" || action.kind === "scroll") {
      steps.push({ action: action.kind });
    }
  }
  return steps.slice(0, MAX_COMPUTER_POINTER_STEPS);
}

export function computerPointerPayload(
  screen: ComputerScreenSize,
  steps: ComputerPointerStep[],
): ComputerPointerPayload | undefined {
  if (steps.length === 0) return undefined;
  return { kind: "pointer", screen, steps: steps.slice(0, MAX_COMPUTER_POINTER_STEPS) };
}

/**
 * Remembers where the page browser last saw each element on screen, so a browser_act batch
 * can be announced as pointer steps before it runs. Geometry never reaches the model.
 */
export class BrowserPointerTargets {
  private boxes = new Map<string, BrowserScreenBox>();
  private screen: ComputerScreenSize | undefined;

  remember(result: { elements?: BrowserSnapshotNode[]; screen?: ComputerScreenSize }) {
    if (!Array.isArray(result.elements)) return;
    this.boxes = new Map();
    this.screen = validScreen(result.screen);
    for (const element of result.elements) {
      if (validBox(element.box)) this.boxes.set(element.ref, element.box);
    }
  }

  forget() {
    this.boxes = new Map();
    this.screen = undefined;
  }

  navigationPayload(): ComputerPointerPayload {
    const screen = this.screen ?? DEFAULT_COMPUTER_SCREEN;
    return {
      kind: "pointer",
      screen,
      steps: [{ action: "move", x: Math.round(screen.width / 2), y: 48 }],
    };
  }

  payloadFor(actions: BrowserActStep[]): ComputerPointerPayload | undefined {
    if (!this.screen) return undefined;
    const steps: ComputerPointerStep[] = [];
    for (const action of actions) {
      if (action.kind === "wait") continue;
      if (action.kind === "scroll" || action.kind === "press") {
        const pointerAction = action.kind === "scroll" ? "scroll" : "key";
        if (action.ref) {
          const box = this.boxes.get(action.ref);
          if (!box) break;
          const [x, y, width, height] = box;
          steps.push({
            action: pointerAction,
            x: Math.round(x + width / 2),
            y: Math.round(y + height / 2),
          });
        } else {
          steps.push({ action: pointerAction });
        }
        continue;
      }
      const box = this.boxes.get(action.ref);
      // Later steps may target elements an earlier step reveals; stop at the first unknown one.
      if (!box) break;
      const [x, y, width, height] = box;
      const pointerAction =
        action.kind === "click" || action.kind === "select"
          ? "click"
          : action.kind === "hover"
            ? "move"
            : "type";
      steps.push({
        action: pointerAction,
        x: Math.round(x + width / 2),
        y: Math.round(y + height / 2),
      });
    }
    return computerPointerPayload(this.screen, steps);
  }
}

/** Strips screen geometry from page-browser results before they reach the model. */
export function withoutBrowserGeometry<T extends { elements?: BrowserSnapshotNode[] }>(
  result: T,
): Omit<T, "screen"> {
  const { screen: _screen, ...rest } = result as T & { screen?: unknown };
  if (!Array.isArray(rest.elements)) return rest;
  return {
    ...rest,
    elements: rest.elements.map(({ box: _box, ...element }) => element),
  };
}

function validScreen(value: unknown): ComputerScreenSize | undefined {
  if (!value || typeof value !== "object") return undefined;
  const { width, height } = value as Record<string, unknown>;
  if (!isPixel(width) || !isPixel(height) || width === 0 || height === 0) return undefined;
  return { width, height };
}

function validBox(value: unknown): value is BrowserScreenBox {
  return (
    Array.isArray(value) &&
    value.length === 4 &&
    value.every(isPixel) &&
    (value[2] as number) > 0 &&
    (value[3] as number) > 0
  );
}

function isPixel(value: unknown): value is number {
  return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100_000;
}