import type { AdapterContext, ComputerRef } from "@rakazo/adapter-kit"; import { describe, expect, it } from "vitest"; import { browserActFromTool, browserNavigateFromTool, browserSnapshotFromTool, parseBrowserActions, } from "./browser-tools.js"; import { filterPageBrowserTools, PAGE_BROWSER_TOOL_NAMES } from "./executor-tools.js"; import { FakeBrowserProvider } from "./fake-browser.js"; const context: AdapterContext = { operationId: "1", traceId: "1", spaceId: "w", userId: "u", signal: new AbortController().signal, }; const computer: ComputerRef = { id: "c1", botId: "b1", kind: "fake", providerRef: "c1", }; describe("browser tools", () => { it("parses click/fill/type/hover/select/press/scroll/wait actions and rejects bad batches", () => { expect( parseBrowserActions([ { kind: "fill", ref: "e1", text: "hi" }, { kind: "click", ref: "e2" }, { kind: "hover", ref: "e2" }, { kind: "select", ref: "e3", text: "Option A" }, { kind: "press", key: "Enter", ref: "e1" }, { kind: "scroll", direction: "down", amount: 4 }, { kind: "wait", ms: 250 }, ]), ).toEqual([ { kind: "fill", ref: "e1", text: "hi" }, { kind: "click", ref: "e2" }, { kind: "hover", ref: "e2" }, { kind: "select", ref: "e3", text: "Option A" }, { kind: "press", key: "Enter", ref: "e1" }, { kind: "scroll", direction: "down", amount: 4 }, { kind: "wait", ms: 250 }, ]); expect(() => parseBrowserActions([])).toThrow(/at least one/i); expect(() => parseBrowserActions([{ kind: "click" }])).toThrow(/ref/i); expect(() => parseBrowserActions([{ kind: "fill", ref: "e1" }])).toThrow(/text/i); expect(() => parseBrowserActions([{ kind: "select", ref: "e1" }])).toThrow(/text/i); expect(() => parseBrowserActions([{ kind: "press" }])).toThrow(/key/i); expect(() => parseBrowserActions([{ kind: "wait", ms: 9000 }])).toThrow(/ms/i); }); it("rejects URL credentials before invoking the browser", async () => { const browser = new FakeBrowserProvider(); browser.navigateError = new Error("must not invoke browser"); for (const scheme of ["http", "https"]) { expect( await browserNavigateFromTool(browser, computer, context, { url: `${scheme}://example:fake-password@example.test`, }), ).toMatchObject({ error: expect.stringContaining("credentials") }); } }); it("navigates, snapshots, and acts through the tool helpers", async () => { const browser = new FakeBrowserProvider({ pages: { "https://example.test/x": { title: "Demo", html: `Demo `, }, }, }); const navigated = await browserNavigateFromTool(browser, computer, context, { url: "https://example.test/x", }); expect(navigated).toMatchObject({ title: "Demo" }); expect("fallback" in navigated && navigated.fallback).toBeFalsy(); const snap = await browserSnapshotFromTool(browser, computer, context, {}); expect(snap).toMatchObject({ title: "Demo" }); expect(String((snap as { tree?: string }).tree)).toMatch(/\[e\d+\]/); const nameRef = (snap as { elements: Array<{ ref: string; name: string }> }).elements.find( (el) => el.name.includes("Name"), )!.ref; const acted = await browserActFromTool(browser, computer, context, { actions: [{ kind: "fill", ref: nameRef, text: "Ada" }], }); expect(acted).toMatchObject({ ok: true, completed: 1 }); }); it("toggles checkboxes via click without undoing activation", async () => { const browser = new FakeBrowserProvider({ pages: { "https://example.test/form": { title: "Form", html: `Form `, }, }, }); await browserNavigateFromTool(browser, computer, context, { url: "https://example.test/form" }); const before = await browserSnapshotFromTool(browser, computer, context, {}); const agree = ( before as { elements: Array<{ ref: string; name: string; value?: string }> } ).elements.find((el) => el.name.includes("Agree"))!; expect(agree.value).toMatch(/unchecked|false|^$/i); const acted = await browserActFromTool(browser, computer, context, { actions: [{ kind: "click", ref: agree.ref }], }); expect(acted).toMatchObject({ ok: true }); const after = await browserSnapshotFromTool(browser, computer, context, {}); const toggled = (after as { elements: Array<{ name: string; value?: string }> }).elements.find( (el) => el.name.includes("Agree"), )!; expect(toggled.value).toMatch(/checked|true/i); }); it("falls back to computer_act for link navigation the in-process page cannot load", async () => { const browser = new FakeBrowserProvider({ pages: { "https://example.test/start": { title: "Start", html: `Start Next `, }, }, }); await browserNavigateFromTool(browser, computer, context, { url: "https://example.test/start", }); const snap = await browserSnapshotFromTool(browser, computer, context, {}); const link = (snap as { elements: Array<{ ref: string; name: string }> }).elements.find((el) => el.name.includes("Next"), )!; const acted = await browserActFromTool(browser, computer, context, { actions: [{ kind: "click", ref: link.ref }], }); expect(acted).toMatchObject({ ok: false, fallback: "computer_act", error: expect.stringMatching(/cannot navigate links|computer_act/i), }); }); it("surfaces computer_act fallback from the tool layer", async () => { const browser = new FakeBrowserProvider(); browser.forceFallback = "cannot attach to page"; const result = await browserNavigateFromTool(browser, computer, context, { url: "https://example.test", }); expect(result).toMatchObject({ fallback: "computer_act", note: expect.stringMatching(/computer_act/i), }); }); it("supports select, hover, press, scroll, and wait in in-process page sessions", async () => { const browser = new FakeBrowserProvider({ pages: { "https://example.test/controls": { title: "Controls", html: `Controls
`, }, }, }); const announced: unknown[] = []; const pointer = { targets: { forget: () => undefined, remember: () => undefined, payloadFor: () => undefined, navigationPayload: () => ({ kind: "pointer" as const, screen: { width: 1280, height: 800 }, steps: [{ action: "move" as const, x: 640, y: 48 }], }), }, announce: async (payload: unknown) => { announced.push(payload); }, }; await browserNavigateFromTool( browser, computer, context, { url: "https://example.test/controls" }, pointer as never, ); expect(announced).toHaveLength(1); const snap = (await browserSnapshotFromTool(browser, computer, context, {})) as { elements: Array<{ ref: string; name: string; value?: string }>; }; const searchRef = snap.elements.find((el) => el.name === "Search")!.ref; const planRef = snap.elements.find((el) => el.name === "Plan")!.ref; const menuRef = snap.elements.find((el) => el.name === "Menu")!.ref; const acted = (await browserActFromTool(browser, computer, context, { actions: [ { kind: "hover", ref: menuRef }, { kind: "select", ref: planRef, text: "Pro Plan" }, { kind: "fill", ref: searchRef, text: "evOo" }, { kind: "press", ref: searchRef, key: "Enter" }, { kind: "scroll", direction: "down", amount: 2 }, { kind: "wait", ms: 10 }, ], })) as { ok: boolean; completed: number; elements: Array<{ name: string; value?: string }> }; expect(acted.ok).toBe(true); expect(acted.completed).toBe(6); expect(acted.elements.find((el) => el.name === "Plan")?.value).toBe("pro"); }); it("hides page browser tools when the computer is not graphical", () => { const tools = [...PAGE_BROWSER_TOOL_NAMES].map((name) => ({ name })); expect(filterPageBrowserTools(tools, true)).toHaveLength(3); expect(filterPageBrowserTools(tools, false)).toEqual([]); expect(filterPageBrowserTools([{ name: "computer_act" }, ...tools], false)).toEqual([ { name: "computer_act" }, ]); }); });