cynarAI 49431def83
feat: show the bot's live cursor and make the in-app browser work (#113)
Bots now announce every click and typing step before it runs, and the
web app draws a cursor in the bot's colour that glides to that point
over the live computer screen, ripples on clicks and shows typing.

- contracts: ComputerPointerPayload on the existing thread.computer event
- executor: computer_act and browser_act announce pointer steps
  (best effort); the page-browser helper reports element screen boxes,
  which are stripped before results reach the model
- web: agentPointerFeed + AgentCursor over the side-panel preview and
  the full-window screen, mapped through the noVNC letterbox
- in-app browser: native WebContentsView on desktop (own session, http(s)
  only, clamped bounds, back/forward, reload, DevTools, load errors) and
  a sandboxed iframe on the web; replaces the dead panel, the fake
  50%/40% cursors and the unimplemented devServer bridge


Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi

Co-authored-by: Claude <noreply@anthropic.com>
2026-10-03 19:08:48 +02:00

166 lines
5.8 KiB
TypeScript

import { screenPointInViewer } from "@rakazo/core";
import { Keyboard } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import {
agentPointerFeed,
POINTER_IDLE_MS,
type PointerPoint,
type PointerScreen,
type PointerSegment,
planPointerTimeline,
pointerFrameAt,
} from "../../lib/agent-pointer";
function prefersReducedMotion() {
return (
typeof window.matchMedia === "function" &&
window.matchMedia("(prefers-reduced-motion: reduce)").matches
);
}
/**
* Draws a bot's announced pointer over its live screen: it glides to each target, ripples on
* clicks and shows when it types. Frames are written straight to the DOM so the chat never
* re-renders at animation rate.
*/
export function AgentCursor({
botId,
color,
name,
}: {
botId: string;
color: string;
name: string;
}) {
const layerRef = useRef<HTMLDivElement>(null);
const cursorRef = useRef<HTMLDivElement>(null);
const rippleRef = useRef<HTMLSpanElement>(null);
const [typing, setTyping] = useState(false);
useEffect(() => {
const layer = layerRef.current;
const cursor = cursorRef.current;
const ripple = rippleRef.current;
if (!layer || !cursor || !ripple) return;
const reducedMotion = prefersReducedMotion();
let screen: PointerScreen | null = null;
let segments: PointerSegment[] = [];
let position: PointerPoint | null = null;
let frame = 0;
let idleTimer: ReturnType<typeof setTimeout> | undefined;
let viewer = { width: layer.clientWidth, height: layer.clientHeight };
let shownTyping = false;
const place = (point: PointerPoint, action: string | null, progress: number) => {
if (!screen) return;
const projected = screenPointInViewer(point, viewer, screen);
cursor.style.transform = `translate3d(${projected.x}px, ${projected.y}px, 0)`;
cursor.dataset.action = action ?? "move";
const pressing = action === "click" || action === "double_click" || action === "right_click";
const pulse = action === "double_click" ? (progress * 2) % 1 : progress;
ripple.style.opacity = pressing && !reducedMotion ? String(0.55 * (1 - pulse)) : "0";
ripple.style.transform = `scale(${0.35 + pulse * 1.15})`;
cursor.style.setProperty("--agent-cursor-press", pressing && pulse < 0.35 ? "0.86" : "1");
const nextTyping = action === "type" || action === "key";
if (nextTyping !== shownTyping) {
shownTyping = nextTyping;
setTyping(nextTyping);
}
};
const rest = () => {
clearTimeout(idleTimer);
idleTimer = setTimeout(() => {
cursor.style.opacity = "0";
}, POINTER_IDLE_MS);
};
const tick = () => {
frame = 0;
const now = performance.now();
const state = pointerFrameAt(segments, now, reducedMotion);
if (!state) return;
position = state.point;
place(state.point, state.action?.kind ?? null, state.action?.progress ?? 0);
if (state.done) {
segments = [];
rest();
return;
}
frame = requestAnimationFrame(tick);
};
const unsubscribe = agentPointerFeed.subscribe(botId, (payload) => {
const now = performance.now();
const current = pointerFrameAt(segments, now, reducedMotion)?.point ?? position;
const sameScreen =
screen?.width === payload.screen.width && screen?.height === payload.screen.height;
screen = payload.screen;
// A first announcement starts at its target rather than flying in from a corner.
const firstTarget = payload.steps.find((step) => typeof step.x === "number");
const from =
current && sameScreen
? current
: firstTarget
? { x: firstTarget.x ?? 0, y: firstTarget.y ?? 0 }
: { x: payload.screen.width / 2, y: payload.screen.height / 2 };
segments = planPointerTimeline(from, payload.steps, now);
clearTimeout(idleTimer);
cursor.style.opacity = "1";
if (!frame) frame = requestAnimationFrame(tick);
});
const resize = new ResizeObserver(() => {
viewer = { width: layer.clientWidth, height: layer.clientHeight };
if (position && !frame) place(position, null, 0);
});
resize.observe(layer);
return () => {
unsubscribe();
resize.disconnect();
cancelAnimationFrame(frame);
clearTimeout(idleTimer);
};
}, [botId]);
return (
<div
ref={layerRef}
aria-hidden
data-testid="agent-cursor-layer"
className="pointer-events-none absolute inset-0 overflow-hidden"
>
<div
ref={cursorRef}
data-testid="agent-cursor"
className="absolute top-0 left-0 opacity-0 transition-opacity duration-300 will-change-transform"
style={{ color }}
>
<span
ref={rippleRef}
className="absolute -top-5 -left-5 size-10 rounded-full border-2 border-current opacity-0"
/>
<svg
viewBox="0 0 20 22"
className="relative block size-5 origin-top-left drop-shadow-md transition-transform duration-100"
style={{ transform: "scale(var(--agent-cursor-press, 1))" }}
>
<path
d="M1.5 1.5 L1.5 17.5 L6 13.2 L9.2 20.2 L12.4 18.8 L9.3 12 L15.5 12 Z"
fill="currentColor"
className="stroke-background"
strokeWidth="1.6"
strokeLinejoin="round"
/>
</svg>
<span className="absolute top-5 left-3.5 flex max-w-40 items-center gap-1 rounded-full bg-popover px-2 py-0.5 text-ui-label font-medium whitespace-nowrap text-popover-foreground shadow-sm ring-1 ring-border">
<span className="size-1.5 shrink-0 rounded-full bg-current" style={{ color }} />
<span className="truncate">{name}</span>
{typing ? <Keyboard strokeWidth={1.8} className="size-3 shrink-0" /> : null}
</span>
</div>
</div>
);
}