Bots now announce every click and typing step before it runs, and the web app draws a cursor in the bot's colour that glides to that point over the live computer screen, ripples on clicks and shows typing. - contracts: ComputerPointerPayload on the existing thread.computer event - executor: computer_act and browser_act announce pointer steps (best effort); the page-browser helper reports element screen boxes, which are stripped before results reach the model - web: agentPointerFeed + AgentCursor over the side-panel preview and the full-window screen, mapped through the noVNC letterbox - in-app browser: native WebContentsView on desktop (own session, http(s) only, clamped bounds, back/forward, reload, DevTools, load errors) and a sandboxed iframe on the web; replaces the dead panel, the fake 50%/40% cursors and the unimplemented devServer bridge Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi Co-authored-by: Claude <noreply@anthropic.com>
166 lines
5.8 KiB
TypeScript
166 lines
5.8 KiB
TypeScript
import { screenPointInViewer } from "@rakazo/core";
|
|
import { Keyboard } from "lucide-react";
|
|
import { useEffect, useRef, useState } from "react";
|
|
import {
|
|
agentPointerFeed,
|
|
POINTER_IDLE_MS,
|
|
type PointerPoint,
|
|
type PointerScreen,
|
|
type PointerSegment,
|
|
planPointerTimeline,
|
|
pointerFrameAt,
|
|
} from "../../lib/agent-pointer";
|
|
|
|
function prefersReducedMotion() {
|
|
return (
|
|
typeof window.matchMedia === "function" &&
|
|
window.matchMedia("(prefers-reduced-motion: reduce)").matches
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Draws a bot's announced pointer over its live screen: it glides to each target, ripples on
|
|
* clicks and shows when it types. Frames are written straight to the DOM so the chat never
|
|
* re-renders at animation rate.
|
|
*/
|
|
export function AgentCursor({
|
|
botId,
|
|
color,
|
|
name,
|
|
}: {
|
|
botId: string;
|
|
color: string;
|
|
name: string;
|
|
}) {
|
|
const layerRef = useRef<HTMLDivElement>(null);
|
|
const cursorRef = useRef<HTMLDivElement>(null);
|
|
const rippleRef = useRef<HTMLSpanElement>(null);
|
|
const [typing, setTyping] = useState(false);
|
|
|
|
useEffect(() => {
|
|
const layer = layerRef.current;
|
|
const cursor = cursorRef.current;
|
|
const ripple = rippleRef.current;
|
|
if (!layer || !cursor || !ripple) return;
|
|
const reducedMotion = prefersReducedMotion();
|
|
let screen: PointerScreen | null = null;
|
|
let segments: PointerSegment[] = [];
|
|
let position: PointerPoint | null = null;
|
|
let frame = 0;
|
|
let idleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
let viewer = { width: layer.clientWidth, height: layer.clientHeight };
|
|
let shownTyping = false;
|
|
|
|
const place = (point: PointerPoint, action: string | null, progress: number) => {
|
|
if (!screen) return;
|
|
const projected = screenPointInViewer(point, viewer, screen);
|
|
cursor.style.transform = `translate3d(${projected.x}px, ${projected.y}px, 0)`;
|
|
cursor.dataset.action = action ?? "move";
|
|
const pressing = action === "click" || action === "double_click" || action === "right_click";
|
|
const pulse = action === "double_click" ? (progress * 2) % 1 : progress;
|
|
ripple.style.opacity = pressing && !reducedMotion ? String(0.55 * (1 - pulse)) : "0";
|
|
ripple.style.transform = `scale(${0.35 + pulse * 1.15})`;
|
|
cursor.style.setProperty("--agent-cursor-press", pressing && pulse < 0.35 ? "0.86" : "1");
|
|
const nextTyping = action === "type" || action === "key";
|
|
if (nextTyping !== shownTyping) {
|
|
shownTyping = nextTyping;
|
|
setTyping(nextTyping);
|
|
}
|
|
};
|
|
|
|
const rest = () => {
|
|
clearTimeout(idleTimer);
|
|
idleTimer = setTimeout(() => {
|
|
cursor.style.opacity = "0";
|
|
}, POINTER_IDLE_MS);
|
|
};
|
|
|
|
const tick = () => {
|
|
frame = 0;
|
|
const now = performance.now();
|
|
const state = pointerFrameAt(segments, now, reducedMotion);
|
|
if (!state) return;
|
|
position = state.point;
|
|
place(state.point, state.action?.kind ?? null, state.action?.progress ?? 0);
|
|
if (state.done) {
|
|
segments = [];
|
|
rest();
|
|
return;
|
|
}
|
|
frame = requestAnimationFrame(tick);
|
|
};
|
|
|
|
const unsubscribe = agentPointerFeed.subscribe(botId, (payload) => {
|
|
const now = performance.now();
|
|
const current = pointerFrameAt(segments, now, reducedMotion)?.point ?? position;
|
|
const sameScreen =
|
|
screen?.width === payload.screen.width && screen?.height === payload.screen.height;
|
|
screen = payload.screen;
|
|
// A first announcement starts at its target rather than flying in from a corner.
|
|
const firstTarget = payload.steps.find((step) => typeof step.x === "number");
|
|
const from =
|
|
current && sameScreen
|
|
? current
|
|
: firstTarget
|
|
? { x: firstTarget.x ?? 0, y: firstTarget.y ?? 0 }
|
|
: { x: payload.screen.width / 2, y: payload.screen.height / 2 };
|
|
segments = planPointerTimeline(from, payload.steps, now);
|
|
clearTimeout(idleTimer);
|
|
cursor.style.opacity = "1";
|
|
if (!frame) frame = requestAnimationFrame(tick);
|
|
});
|
|
|
|
const resize = new ResizeObserver(() => {
|
|
viewer = { width: layer.clientWidth, height: layer.clientHeight };
|
|
if (position && !frame) place(position, null, 0);
|
|
});
|
|
resize.observe(layer);
|
|
|
|
return () => {
|
|
unsubscribe();
|
|
resize.disconnect();
|
|
cancelAnimationFrame(frame);
|
|
clearTimeout(idleTimer);
|
|
};
|
|
}, [botId]);
|
|
|
|
return (
|
|
<div
|
|
ref={layerRef}
|
|
aria-hidden
|
|
data-testid="agent-cursor-layer"
|
|
className="pointer-events-none absolute inset-0 overflow-hidden"
|
|
>
|
|
<div
|
|
ref={cursorRef}
|
|
data-testid="agent-cursor"
|
|
className="absolute top-0 left-0 opacity-0 transition-opacity duration-300 will-change-transform"
|
|
style={{ color }}
|
|
>
|
|
<span
|
|
ref={rippleRef}
|
|
className="absolute -top-5 -left-5 size-10 rounded-full border-2 border-current opacity-0"
|
|
/>
|
|
<svg
|
|
viewBox="0 0 20 22"
|
|
className="relative block size-5 origin-top-left drop-shadow-md transition-transform duration-100"
|
|
style={{ transform: "scale(var(--agent-cursor-press, 1))" }}
|
|
>
|
|
<path
|
|
d="M1.5 1.5 L1.5 17.5 L6 13.2 L9.2 20.2 L12.4 18.8 L9.3 12 L15.5 12 Z"
|
|
fill="currentColor"
|
|
className="stroke-background"
|
|
strokeWidth="1.6"
|
|
strokeLinejoin="round"
|
|
/>
|
|
</svg>
|
|
<span className="absolute top-5 left-3.5 flex max-w-40 items-center gap-1 rounded-full bg-popover px-2 py-0.5 text-ui-label font-medium whitespace-nowrap text-popover-foreground shadow-sm ring-1 ring-border">
|
|
<span className="size-1.5 shrink-0 rounded-full bg-current" style={{ color }} />
|
|
<span className="truncate">{name}</span>
|
|
{typing ? <Keyboard strokeWidth={1.8} className="size-3 shrink-0" /> : null}
|
|
</span>
|
|
</div>
|
|
</div>
|
|
);
|
|
}
|