evOo-local/packages/core/src/teach-recording.test.ts
cynarAI 49431def83
feat: show the bot's live cursor and make the in-app browser work (#113)
Bots now announce every click and typing step before it runs, and the
web app draws a cursor in the bot's colour that glides to that point
over the live computer screen, ripples on clicks and shows typing.

- contracts: ComputerPointerPayload on the existing thread.computer event
- executor: computer_act and browser_act announce pointer steps
  (best effort); the page-browser helper reports element screen boxes,
  which are stripped before results reach the model
- web: agentPointerFeed + AgentCursor over the side-panel preview and
  the full-window screen, mapped through the noVNC letterbox
- in-app browser: native WebContentsView on desktop (own session, http(s)
  only, clamped bounds, back/forward, reload, DevTools, load errors) and
  a sandboxed iframe on the web; replaces the dead panel, the fake
  50%/40% cursors and the unimplemented devServer bridge


Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi

Co-authored-by: Claude <noreply@anthropic.com>
2026-10-03 19:08:48 +02:00

110 lines
4.4 KiB
TypeScript

import { describe, expect, it } from "vitest";
import {
computerInputForDomKey,
computerScreenSize,
mapTeachPointer,
screenPointInViewer,
teachCaptureKey,
} from "./teach-recording.js";
describe("teachCaptureKey", () => {
it("records printable characters and editing keys", () => {
expect(teachCaptureKey("a")).toBe("a");
expect(teachCaptureKey("Enter")).toBe("Enter");
expect(teachCaptureKey("Backspace")).toBe("Backspace");
expect(teachCaptureKey("Tab")).toBe("Tab");
expect(teachCaptureKey("ArrowLeft")).toBe("ArrowLeft");
});
it("ignores modifiers and unmapped keys", () => {
expect(teachCaptureKey("a", { metaKey: true })).toBeNull();
expect(teachCaptureKey("Shift")).toBeNull();
expect(teachCaptureKey("F5")).toBeNull();
});
});
describe("computerInputForDomKey", () => {
it("types printable characters instead of pressing a keysym", () => {
expect(computerInputForDomKey("a")).toEqual({ kind: "clipboard", text: "a" });
expect(computerInputForDomKey(" ")).toEqual({ kind: "clipboard", text: " " });
expect(computerInputForDomKey("@")).toEqual({ kind: "clipboard", text: "@" });
expect(computerInputForDomKey(".")).toEqual({ kind: "clipboard", text: "." });
});
it("translates browser key names to X11 keysyms", () => {
expect(computerInputForDomKey("Enter")).toEqual({ kind: "key", key: "Return" });
expect(computerInputForDomKey("Backspace")).toEqual({ kind: "key", key: "BackSpace" });
expect(computerInputForDomKey("ArrowLeft")).toEqual({ kind: "key", key: "Left" });
expect(computerInputForDomKey("ArrowDown")).toEqual({ kind: "key", key: "Down" });
expect(computerInputForDomKey("Escape")).toEqual({ kind: "key", key: "Escape" });
});
it("passes through names it does not know", () => {
expect(computerInputForDomKey("F5")).toEqual({ kind: "key", key: "F5" });
});
});
describe("computerScreenSize", () => {
it("uses box geometry and the 1280x800 default for other sandboxes", () => {
expect(computerScreenSize("box")).toEqual({ width: 1920, height: 1080 });
expect(computerScreenSize("e2b")).toEqual({ width: 1280, height: 800 });
expect(computerScreenSize("daytona")).toEqual({ width: 1280, height: 800 });
});
});
describe("mapTeachPointer", () => {
it("scales overlay clicks onto the remote screen size", () => {
const rect = { left: 0, top: 0, width: 640, height: 400 };
expect(mapTeachPointer(320, 200, rect, { width: 1280, height: 800 })).toEqual({
x: 640,
y: 400,
});
expect(mapTeachPointer(320, 200, rect, { width: 1920, height: 1080 })).toEqual({
x: 960,
y: 540,
});
});
it("maps through the letterbox when the viewer is wider than the screen", () => {
// 1600x800 viewer, 1280x800 screen: scale 1, 160px bars left and right.
const rect = { left: 0, top: 0, width: 1600, height: 800 };
const screen = { width: 1280, height: 800 };
expect(mapTeachPointer(160, 0, rect, screen)).toEqual({ x: 0, y: 0 });
expect(mapTeachPointer(800, 400, rect, screen)).toEqual({ x: 640, y: 400 });
expect(mapTeachPointer(1440, 799, rect, screen)).toEqual({ x: 1279, y: 799 });
});
it("clamps clicks that land on the letterbox bars", () => {
const rect = { left: 0, top: 0, width: 1600, height: 800 };
const screen = { width: 1280, height: 800 };
expect(mapTeachPointer(0, 400, rect, screen)).toEqual({ x: 0, y: 400 });
expect(mapTeachPointer(1600, 400, rect, screen)).toEqual({ x: 1279, y: 400 });
});
it("accounts for the rect offset", () => {
const rect = { left: 100, top: 50, width: 640, height: 400 };
expect(mapTeachPointer(420, 250, rect, { width: 1280, height: 800 })).toEqual({
x: 640,
y: 400,
});
});
});
describe("screenPointInViewer", () => {
it("projects through the same letterbox the viewer draws", () => {
const screen = { width: 1280, height: 800 };
// A wide viewer pillarboxes the 16:10 screen.
expect(screenPointInViewer({ x: 0, y: 0 }, { width: 1000, height: 400 }, screen)).toEqual({
x: 180,
y: 0,
});
expect(screenPointInViewer({ x: 1280, y: 800 }, { width: 1000, height: 400 }, screen)).toEqual({
x: 820,
y: 400,
});
const rect = { left: 0, top: 0, width: 640, height: 600 };
const projected = screenPointInViewer({ x: 640, y: 400 }, rect, screen);
expect(mapTeachPointer(projected.x, projected.y, rect, screen)).toEqual({ x: 640, y: 400 });
});
});