evOo-local/packages/adapters/src/browser-tools.test.ts

import type { AdapterContext, ComputerRef } from "@rakazo/adapter-kit";
import { describe, expect, it } from "vitest";
import {
  browserActFromTool,
  browserNavigateFromTool,
  browserSnapshotFromTool,
  parseBrowserActions,
} from "./browser-tools.js";
import { filterPageBrowserTools, PAGE_BROWSER_TOOL_NAMES } from "./executor-tools.js";
import { FakeBrowserProvider } from "./fake-browser.js";

const context: AdapterContext = {
  operationId: "1",
  traceId: "1",
  spaceId: "w",
  userId: "u",
  signal: new AbortController().signal,
};

const computer: ComputerRef = {
  id: "c1",
  botId: "b1",
  kind: "fake",
  providerRef: "c1",
};

describe("browser tools", () => {
  it("parses click/fill/type/hover/select/press/scroll/wait actions and rejects bad batches", () => {
    expect(
      parseBrowserActions([
        { kind: "fill", ref: "e1", text: "hi" },
        { kind: "click", ref: "e2" },
        { kind: "hover", ref: "e2" },
        { kind: "select", ref: "e3", text: "Option A" },
        { kind: "press", key: "Enter", ref: "e1" },
        { kind: "scroll", direction: "down", amount: 4 },
        { kind: "wait", ms: 250 },
      ]),
    ).toEqual([
      { kind: "fill", ref: "e1", text: "hi" },
      { kind: "click", ref: "e2" },
      { kind: "hover", ref: "e2" },
      { kind: "select", ref: "e3", text: "Option A" },
      { kind: "press", key: "Enter", ref: "e1" },
      { kind: "scroll", direction: "down", amount: 4 },
      { kind: "wait", ms: 250 },
    ]);
    expect(() => parseBrowserActions([])).toThrow(/at least one/i);
    expect(() => parseBrowserActions([{ kind: "click" }])).toThrow(/ref/i);
    expect(() => parseBrowserActions([{ kind: "fill", ref: "e1" }])).toThrow(/text/i);
    expect(() => parseBrowserActions([{ kind: "select", ref: "e1" }])).toThrow(/text/i);
    expect(() => parseBrowserActions([{ kind: "press" }])).toThrow(/key/i);
    expect(() => parseBrowserActions([{ kind: "wait", ms: 9000 }])).toThrow(/ms/i);
  });

  it("rejects URL credentials before invoking the browser", async () => {
    const browser = new FakeBrowserProvider();
    browser.navigateError = new Error("must not invoke browser");
    for (const scheme of ["http", "https"]) {
      expect(
        await browserNavigateFromTool(browser, computer, context, {
          url: `${scheme}://example:fake-password@example.test`,
        }),
      ).toMatchObject({ error: expect.stringContaining("credentials") });
    }
  });

  it("navigates, snapshots, and acts through the tool helpers", async () => {
    const browser = new FakeBrowserProvider({
      pages: {
        "https://example.test/x": {
          title: "Demo",
          html: `<!doctype html><html><head><title>Demo</title></head><body>
            <input aria-label="Name" />
            <button>Save</button>
          </body></html>`,
        },
      },
    });
    const navigated = await browserNavigateFromTool(browser, computer, context, {
      url: "https://example.test/x",
    });
    expect(navigated).toMatchObject({ title: "Demo" });
    expect("fallback" in navigated && navigated.fallback).toBeFalsy();

    const snap = await browserSnapshotFromTool(browser, computer, context, {});
    expect(snap).toMatchObject({ title: "Demo" });
    expect(String((snap as { tree?: string }).tree)).toMatch(/\[e\d+\]/);

    const nameRef = (snap as { elements: Array<{ ref: string; name: string }> }).elements.find(
      (el) => el.name.includes("Name"),
    )!.ref;
    const acted = await browserActFromTool(browser, computer, context, {
      actions: [{ kind: "fill", ref: nameRef, text: "Ada" }],
    });
    expect(acted).toMatchObject({ ok: true, completed: 1 });
  });

  it("toggles checkboxes via click without undoing activation", async () => {
    const browser = new FakeBrowserProvider({
      pages: {
        "https://example.test/form": {
          title: "Form",
          html: `<!doctype html><html><head><title>Form</title></head><body>
            <label><input type="checkbox" aria-label="Agree" /> Agree</label>
          </body></html>`,
        },
      },
    });
    await browserNavigateFromTool(browser, computer, context, { url: "https://example.test/form" });
    const before = await browserSnapshotFromTool(browser, computer, context, {});
    const agree = (
      before as { elements: Array<{ ref: string; name: string; value?: string }> }
    ).elements.find((el) => el.name.includes("Agree"))!;
    expect(agree.value).toMatch(/unchecked|false|^$/i);
    const acted = await browserActFromTool(browser, computer, context, {
      actions: [{ kind: "click", ref: agree.ref }],
    });
    expect(acted).toMatchObject({ ok: true });
    const after = await browserSnapshotFromTool(browser, computer, context, {});
    const toggled = (after as { elements: Array<{ name: string; value?: string }> }).elements.find(
      (el) => el.name.includes("Agree"),
    )!;
    expect(toggled.value).toMatch(/checked|true/i);
  });

  it("falls back to computer_act for link navigation the in-process page cannot load", async () => {
    const browser = new FakeBrowserProvider({
      pages: {
        "https://example.test/start": {
          title: "Start",
          html: `<!doctype html><html><head><title>Start</title></head><body>
            <a href="https://example.test/next">Next</a>
          </body></html>`,
        },
      },
    });
    await browserNavigateFromTool(browser, computer, context, {
      url: "https://example.test/start",
    });
    const snap = await browserSnapshotFromTool(browser, computer, context, {});
    const link = (snap as { elements: Array<{ ref: string; name: string }> }).elements.find((el) =>
      el.name.includes("Next"),
    )!;
    const acted = await browserActFromTool(browser, computer, context, {
      actions: [{ kind: "click", ref: link.ref }],
    });
    expect(acted).toMatchObject({
      ok: false,
      fallback: "computer_act",
      error: expect.stringMatching(/cannot navigate links|computer_act/i),
    });
  });

  it("surfaces computer_act fallback from the tool layer", async () => {
    const browser = new FakeBrowserProvider();
    browser.forceFallback = "cannot attach to page";
    const result = await browserNavigateFromTool(browser, computer, context, {
      url: "https://example.test",
    });
    expect(result).toMatchObject({
      fallback: "computer_act",
      note: expect.stringMatching(/computer_act/i),
    });
  });

  it("supports select, hover, press, scroll, and wait in in-process page sessions", async () => {
    const browser = new FakeBrowserProvider({
      pages: {
        "https://example.test/controls": {
          title: "Controls",
          html: `<!doctype html><html><head><title>Controls</title></head><body>
            <form id="f" onsubmit="this.dataset.submitted='yes'; return false;">
              <input aria-label="Search" />
              <select aria-label="Plan">
                <option value="free">Free</option>
                <option value="pro">Pro Plan</option>
              </select>
              <button type="button" onmouseenter="this.dataset.hovered='yes'">Menu</button>
            </form>
          </body></html>`,
        },
      },
    });
    const announced: unknown[] = [];
    const pointer = {
      targets: {
        forget: () => undefined,
        remember: () => undefined,
        payloadFor: () => undefined,
        navigationPayload: () => ({
          kind: "pointer" as const,
          screen: { width: 1280, height: 800 },
          steps: [{ action: "move" as const, x: 640, y: 48 }],
        }),
      },
      announce: async (payload: unknown) => {
        announced.push(payload);
      },
    };
    await browserNavigateFromTool(
      browser,
      computer,
      context,
      { url: "https://example.test/controls" },
      pointer as never,
    );
    expect(announced).toHaveLength(1);

    const snap = (await browserSnapshotFromTool(browser, computer, context, {})) as {
      elements: Array<{ ref: string; name: string; value?: string }>;
    };
    const searchRef = snap.elements.find((el) => el.name === "Search")!.ref;
    const planRef = snap.elements.find((el) => el.name === "Plan")!.ref;
    const menuRef = snap.elements.find((el) => el.name === "Menu")!.ref;

    const acted = (await browserActFromTool(browser, computer, context, {
      actions: [
        { kind: "hover", ref: menuRef },
        { kind: "select", ref: planRef, text: "Pro Plan" },
        { kind: "fill", ref: searchRef, text: "evOo" },
        { kind: "press", ref: searchRef, key: "Enter" },
        { kind: "scroll", direction: "down", amount: 2 },
        { kind: "wait", ms: 10 },
      ],
    })) as { ok: boolean; completed: number; elements: Array<{ name: string; value?: string }> };
    expect(acted.ok).toBe(true);
    expect(acted.completed).toBe(6);
    expect(acted.elements.find((el) => el.name === "Plan")?.value).toBe("pro");
  });

  it("hides page browser tools when the computer is not graphical", () => {
    const tools = [...PAGE_BROWSER_TOOL_NAMES].map((name) => ({ name }));
    expect(filterPageBrowserTools(tools, true)).toHaveLength(3);
    expect(filterPageBrowserTools(tools, false)).toEqual([]);
    expect(filterPageBrowserTools([{ name: "computer_act" }, ...tools], false)).toEqual([
      { name: "computer_act" },
    ]);
  });
});