evOo-local/packages/adapters/src/computer-pointer.ts

import type {
  BrowserActStep,
  BrowserScreenBox,
  BrowserSnapshotNode,
  ComputerAction,
} from "@rakazo/adapter-kit";
import {
  type ComputerPointerPayload,
  type ComputerPointerStep,
  MAX_COMPUTER_POINTER_STEPS,
} from "@rakazo/contracts";

export type ComputerScreenSize = ComputerPointerPayload["screen"];

/** Pointer steps for a validated computer_act batch, in execution order. */
export function computerPointerSteps(actions: ComputerAction[]): ComputerPointerStep[] {
  const steps: ComputerPointerStep[] = [];
  for (let index = 0; index < actions.length; index += 1) {
    const action = actions[index]!;
    if (action.kind === "pointer") {
      const next = actions[index + 1];
      // parseComputerActions expands a double click into two identical clicks.
      if (
        action.type === "click" &&
        next?.kind === "pointer" &&
        next.type === "click" &&
        next.x === action.x &&
        next.y === action.y &&
        next.button === action.button
      ) {
        steps.push({ action: "double_click", x: action.x, y: action.y });
        index += 1;
        continue;
      }
      const kind =
        action.type === "click"
          ? action.button === "right"
            ? "right_click"
            : "click"
          : action.type;
      steps.push({ action: kind, x: action.x, y: action.y });
    } else if (action.kind === "clipboard") {
      steps.push({ action: "type" });
    } else if (action.kind === "key" || action.kind === "scroll") {
      steps.push({ action: action.kind });
    }
  }
  return steps.slice(0, MAX_COMPUTER_POINTER_STEPS);
}

export function computerPointerPayload(
  screen: ComputerScreenSize,
  steps: ComputerPointerStep[],
): ComputerPointerPayload | undefined {
  if (steps.length === 0) return undefined;
  return { kind: "pointer", screen, steps: steps.slice(0, MAX_COMPUTER_POINTER_STEPS) };
}

/**
 * Remembers where the page browser last saw each element on screen, so a browser_act batch
 * can be announced as pointer steps before it runs. Geometry never reaches the model.
 */
export class BrowserPointerTargets {
  private boxes = new Map<string, BrowserScreenBox>();
  private screen: ComputerScreenSize | undefined;

  remember(result: { elements?: BrowserSnapshotNode[]; screen?: ComputerScreenSize }) {
    if (!Array.isArray(result.elements)) return;
    this.boxes = new Map();
    this.screen = validScreen(result.screen);
    for (const element of result.elements) {
      if (validBox(element.box)) this.boxes.set(element.ref, element.box);
    }
  }

  forget() {
    this.boxes = new Map();
    this.screen = undefined;
  }

  payloadFor(actions: BrowserActStep[]): ComputerPointerPayload | undefined {
    if (!this.screen) return undefined;
    const steps: ComputerPointerStep[] = [];
    for (const action of actions) {
      const box = this.boxes.get(action.ref);
      // Later steps may target elements an earlier step reveals; stop at the first unknown one.
      if (!box) break;
      const [x, y, width, height] = box;
      steps.push({
        action: action.kind === "click" ? "click" : "type",
        x: Math.round(x + width / 2),
        y: Math.round(y + height / 2),
      });
    }
    return computerPointerPayload(this.screen, steps);
  }
}

/** Strips screen geometry from page-browser results before they reach the model. */
export function withoutBrowserGeometry<T extends { elements?: BrowserSnapshotNode[] }>(
  result: T,
): Omit<T, "screen"> {
  const { screen: _screen, ...rest } = result as T & { screen?: unknown };
  if (!Array.isArray(rest.elements)) return rest;
  return {
    ...rest,
    elements: rest.elements.map(({ box: _box, ...element }) => element),
  };
}

function validScreen(value: unknown): ComputerScreenSize | undefined {
  if (!value || typeof value !== "object") return undefined;
  const { width, height } = value as Record<string, unknown>;
  if (!isPixel(width) || !isPixel(height) || width === 0 || height === 0) return undefined;
  return { width, height };
}

function validBox(value: unknown): value is BrowserScreenBox {
  return (
    Array.isArray(value) &&
    value.length === 4 &&
    value.every(isPixel) &&
    (value[2] as number) > 0 &&
    (value[3] as number) > 0
  );
}

function isPixel(value: unknown): value is number {
  return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100_000;
}