49431def83
Bots now announce every click and typing step before it runs, and the web app draws a cursor in the bot's colour that glides to that point over the live computer screen, ripples on clicks and shows typing. - contracts: ComputerPointerPayload on the existing thread.computer event - executor: computer_act and browser_act announce pointer steps (best effort); the page-browser helper reports element screen boxes, which are stripped before results reach the model - web: agentPointerFeed + AgentCursor over the side-panel preview and the full-window screen, mapped through the noVNC letterbox - in-app browser: native WebContentsView on desktop (own session, http(s) only, clamped bounds, back/forward, reload, DevTools, load errors) and a sandboxed iframe on the web; replaces the dead panel, the fake 50%/40% cursors and the unimplemented devServer bridge Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi Co-authored-by: Claude <noreply@anthropic.com>
89 lines
3.2 KiB
TypeScript
89 lines
3.2 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import {
|
|
agentPointerFeed,
|
|
moveDurationMs,
|
|
planPointerTimeline,
|
|
pointerFrameAt,
|
|
} from "./agent-pointer";
|
|
|
|
const payload = {
|
|
kind: "pointer",
|
|
screen: { width: 1280, height: 800 },
|
|
steps: [{ action: "click", x: 100, y: 100 }],
|
|
};
|
|
|
|
describe("agentPointerFeed", () => {
|
|
it("delivers valid live announcements only to the bot's viewers", () => {
|
|
const seen: unknown[] = [];
|
|
const other: unknown[] = [];
|
|
const stop = agentPointerFeed.subscribe("bot-a", (value) => seen.push(value));
|
|
const stopOther = agentPointerFeed.subscribe("bot-b", (value) => other.push(value));
|
|
try {
|
|
const now = Date.parse("2026-01-01T00:00:10Z");
|
|
expect(agentPointerFeed.publish("bot-a", payload, "2026-01-01T00:00:05Z", now)).toBe(true);
|
|
// Replayed history and malformed payloads never move the cursor.
|
|
expect(agentPointerFeed.publish("bot-a", payload, "2025-12-31T23:00:00Z", now)).toBe(false);
|
|
expect(agentPointerFeed.publish("bot-a", { ...payload, steps: [] }, undefined, now)).toBe(
|
|
false,
|
|
);
|
|
expect(agentPointerFeed.publish("bot-a", { kind: "status" }, undefined, now)).toBe(false);
|
|
expect(seen).toEqual([payload]);
|
|
expect(other).toEqual([]);
|
|
} finally {
|
|
stop();
|
|
stopOther();
|
|
}
|
|
expect(agentPointerFeed.publish("bot-a", payload)).toBe(true);
|
|
expect(seen).toHaveLength(1);
|
|
});
|
|
});
|
|
|
|
describe("pointer timeline", () => {
|
|
it("travels, acts in place for keyboard steps, then rests", () => {
|
|
const segments = planPointerTimeline(
|
|
{ x: 0, y: 0 },
|
|
[{ action: "click", x: 300, y: 400 }, { action: "type" }, { action: "move", x: 300, y: 400 }],
|
|
1_000,
|
|
);
|
|
expect(segments.map((segment) => segment.to)).toEqual([
|
|
{ x: 300, y: 400 },
|
|
{ x: 300, y: 400 },
|
|
{ x: 300, y: 400 },
|
|
]);
|
|
const [click, type, move] = segments;
|
|
expect(click!.arrive - click!.start).toBe(moveDurationMs(500));
|
|
expect(type!.arrive).toBe(type!.start);
|
|
expect(move!.end).toBe(move!.start);
|
|
|
|
const midway = pointerFrameAt(segments, (click!.start + click!.arrive) / 2)!;
|
|
expect(midway.action).toBeNull();
|
|
// The bow lifts the cursor off the straight line between the points.
|
|
const onLine = { x: 150, y: 200 };
|
|
expect(Math.hypot(midway.point.x - onLine.x, midway.point.y - onLine.y)).toBeGreaterThan(10);
|
|
|
|
const clicking = pointerFrameAt(segments, click!.arrive + 1)!;
|
|
expect(clicking.point).toEqual({ x: 300, y: 400 });
|
|
expect(clicking.action?.kind).toBe("click");
|
|
|
|
const typing = pointerFrameAt(segments, type!.arrive + 10)!;
|
|
expect(typing.action?.kind).toBe("type");
|
|
|
|
expect(pointerFrameAt(segments, move!.end + 1)).toEqual({
|
|
point: { x: 300, y: 400 },
|
|
action: null,
|
|
done: true,
|
|
});
|
|
});
|
|
|
|
it("jumps straight to the target when motion is reduced", () => {
|
|
const segments = planPointerTimeline({ x: 0, y: 0 }, [{ action: "move", x: 50, y: 60 }], 0);
|
|
expect(pointerFrameAt(segments, 1, true)?.point).toEqual({ x: 50, y: 60 });
|
|
});
|
|
|
|
it("bounds travel time so long and short moves both feel responsive", () => {
|
|
expect(moveDurationMs(0)).toBe(0);
|
|
expect(moveDurationMs(5)).toBe(180);
|
|
expect(moveDurationMs(10_000)).toBe(650);
|
|
});
|
|
});
|