cynarAI 49431def83
feat: show the bot's live cursor and make the in-app browser work (#113)
Bots now announce every click and typing step before it runs, and the
web app draws a cursor in the bot's colour that glides to that point
over the live computer screen, ripples on clicks and shows typing.

- contracts: ComputerPointerPayload on the existing thread.computer event
- executor: computer_act and browser_act announce pointer steps
  (best effort); the page-browser helper reports element screen boxes,
  which are stripped before results reach the model
- web: agentPointerFeed + AgentCursor over the side-panel preview and
  the full-window screen, mapped through the noVNC letterbox
- in-app browser: native WebContentsView on desktop (own session, http(s)
  only, clamped bounds, back/forward, reload, DevTools, load errors) and
  a sandboxed iframe on the web; replaces the dead panel, the fake
  50%/40% cursors and the unimplemented devServer bridge


Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi

Co-authored-by: Claude <noreply@anthropic.com>
2026-10-03 19:08:48 +02:00

195 lines
6.6 KiB
TypeScript

/**
* `unsupported` covers an unpackaged build and a repository with no published releases, which is
* the normal state for a fork. It is not an error the user needs to act on. Automatic checks stay
* frozen after an empty feed; a manual check may retry when the install itself supports updates.
*/
export type DesktopUpdatePhase =
| "idle"
| "checking"
| "available"
| "downloading"
| "ready"
| "unsupported"
| "error";
export interface DesktopUpdateState {
phase: DesktopUpdatePhase;
/** The installed desktop release, which can drift from the server this app points at. */
currentVersion: string;
availableVersion: string | null;
/** Download progress 0-100, only while `downloading`. */
percent: number | null;
message: string | null;
checkedAt: string | null;
}
export interface RakazoDesktopUpdate {
state: () => Promise<DesktopUpdateState>;
check: () => Promise<DesktopUpdateState>;
download: () => Promise<DesktopUpdateState>;
/** Quits and relaunches into the downloaded release; only useful once `phase` is `ready`. */
install: () => Promise<DesktopUpdateState>;
}
export interface RakazoDesktopOAuthCallback {
code: string;
state?: string;
}
export interface RakazoDesktopBrowserPreviewState {
url: string;
title: string;
loading: boolean;
canGoBack: boolean;
canGoForward: boolean;
/** The last main-frame load failure for `url`, cleared by the next successful load. */
error: string | null;
}
/** Rectangle in CSS pixels of the app window's content area. */
export interface RakazoDesktopBrowserBounds {
x: number;
y: number;
width: number;
height: number;
}
/**
* Optional desktop bridge for the in-app browser: a native Chromium view the main process lays
* over the panel the renderer reserves for it. Absent in the browser, where an iframe is used.
*/
export interface RakazoDesktopBrowserPreview {
/** Shows the view over these bounds; call again whenever the reserved area moves. */
show: (bounds: RakazoDesktopBrowserBounds) => Promise<void>;
hide: () => Promise<void>;
/** Only http(s) URLs load; anything else is rejected. */
navigate: (url: string) => Promise<void>;
back: () => Promise<void>;
forward: () => Promise<void>;
reload: () => Promise<void>;
stop: () => Promise<void>;
openDevTools: () => Promise<void>;
state: () => Promise<RakazoDesktopBrowserPreviewState>;
onState: (listener: (state: RakazoDesktopBrowserPreviewState) => void) => () => void;
}
export interface RakazoDesktop {
/** Only the isolated local settings window is authorized to call this bridge. */
localSettings?: {
request: (pathname: string, body: string) => Promise<{ status: number; body: string }>;
};
files?: {
/** Opens a native folder picker and returns an absolute host path, or null when cancelled. */
pickDirectory: () => Promise<string | null>;
};
platform: string;
window: {
close: () => Promise<void>;
minimize: () => Promise<void>;
toggleMaximize: () => Promise<void>;
state: () => Promise<{ minimized: boolean; maximized: boolean; fullScreen: boolean }>;
};
update: RakazoDesktopUpdate;
oauth: {
/**
* Open system-browser auth. A redirect_uri must be HTTP loopback with state;
* URLs without a redirect use backend polling. Optional for older desktops.
*/
open?: (authorizationUrl: string) => Promise<void>;
cancel?: (authorizationUrl: string) => Promise<void>;
/**
* Authorization codes captured from the system browser or a legacy popup.
* Returns an unsubscribe function.
*/
onCallback: (listener: (callback: RakazoDesktopOAuthCallback) => void) => () => void;
};
/** Present only when the desktop shell hosts an embedded browser preview. */
browserPreview?: RakazoDesktopBrowserPreview;
}
/**
* How the desktop app was pointed at a Rakazo server during first-run setup.
* `new` is the Docker Compose stack this app installs and runs on the same computer.
*/
export type DesktopInstanceMode = "new" | "existing";
export interface DesktopSetup {
mode: DesktopInstanceMode;
serverUrl: string;
}
export interface DesktopSetupState {
defaultLocalUrl: string;
saved: DesktopSetup | null;
/** Present when a saved or newly selected server could not be reopened. */
error?: string;
}
export interface DesktopReachability {
ok: boolean;
/** HTTP status when the server answered, absent when it could not be reached. */
status?: number;
/** Normalized URL that was probed, absent when the input was not a usable URL. */
url?: string;
error?: string;
}
export interface DesktopStackProbeResponse {
ok: true;
imageTag: string;
}
/**
* Lifecycle of the Docker Compose stack the desktop app manages for mode `new`.
* `docker-missing` and `docker-not-running` wait for the person to act; `ready` and
* `failed` are terminal until the next start.
*/
export type DesktopLocalStackPhase =
| "idle"
| "checking-docker"
| "docker-missing"
| "docker-not-running"
| "preparing"
| "pulling"
| "starting"
| "waiting-healthy"
| "ready"
| "failed";
export interface DesktopLocalStackState {
phase: DesktopLocalStackPhase;
/** One actionable sentence; null while the stack is progressing normally. */
message: string | null;
/** Bounded tail of docker output for the current attempt. */
output: string[];
/** Bytes pulled per image layer in the current attempt; the sum is the only progress Docker reports. */
layerBytes: Record<string, number>;
/** Image tag this app launches (`v<app version>` for installed builds, `edge` otherwise). */
imageTag: string;
}
export type DesktopSetupLink = "docker-desktop" | "orbstack" | "docker-engine";
/**
* Bridge exposed only to the first-run setup window. The app window keeps the
* narrower `rakazoDesktop` bridge so a connected server can never re-point the app.
*/
export interface RakazoSetup {
/** Used only to reserve space for native window controls in the local setup UI. */
platform: string;
state: () => Promise<DesktopSetupState>;
test: (url: string) => Promise<DesktopReachability>;
save: (setup: DesktopSetup) => Promise<{ ok: boolean; error?: string }>;
quit: () => Promise<void>;
/** Opens one of the Docker install pages in the system browser. */
openLink: (link: DesktopSetupLink) => Promise<void>;
/** The Docker Compose stack this app installs and runs for mode `new`. */
stack: {
state: () => Promise<DesktopLocalStackState>;
/** Starts (or retries) the stack; a no-op while a start is already in flight. */
start: () => Promise<DesktopLocalStackState>;
/** Fires on every state change so progress never depends on a renderer timer. */
onChange: (listener: (state: DesktopLocalStackState) => void) => void;
};
}