Compare commits

..

3 Commits

Author SHA1 Message Date
Claude
b38e410fa3
Merge remote-tracking branch 'origin/main' into claude/amazing-thompson-bgplnq
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SSVQGR4WU92LaddTP19GcQ
2026-10-03 15:12:07 +00:00
Claude
2736380934
docs(agents): make routine calls without asking
Agents in this repository stopped to ask the maintainer about routine
steps such as reopening a pull request or linking CI screenshots. The
rule asks them to decide these themselves, verify the result and report
it, and to ask only before irreversible actions or for product decisions
without a sensible default.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SSVQGR4WU92LaddTP19GcQ
2026-10-03 14:58:49 +00:00
Claude
53ec88ac7d
fix(web): keep secondary shell text readable in the light theme
Secondary shell text was muted text dimmed further with opacity: the
section labels, the projects caption, the sidebar "+" icons, the privacy
panel, the trading subtitle and the Heute brief's Mission Control link.
Muted text measures 4.4:1 on the light sidebar, so these fell to 1.9 to
3.4:1 in the light theme; a few were below 4.5:1 in dark as well.

- Secondary sidebar text and icons share `rk-sidebar-muted`, the sidebar
  foreground at 65% that row times already used (5.0:1 on the light
  sidebar). Row times now use the class too.
- Section labels and space or project group titles use the dimmed
  foreground in light and keep the 80% muted color in dark; group titles
  move up from 60%, which was 3.4:1 in dark.
- The privacy panel heading is plain text with a success-colored check
  instead of emerald text, and its details pass 4.5:1 in both themes.
- Needs-you chips keep the warning tint and border; their label uses the
  foreground color in light.
- The Mission Control link uses the full muted color.
- E2E: the shell test checks every text in the sidebar except the logo,
  and the Heute brief, in both themes, and captures
  `shell-contrast-{light,dark}-heute`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SSVQGR4WU92LaddTP19GcQ
2026-10-03 14:58:48 +00:00
49 changed files with 418 additions and 2639 deletions

View File

@ -263,8 +263,8 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
owner_type="$(gh api "repos/${GITHUB_REPOSITORY}" --jq '.owner.type')"
private="$(gh api "repos/${GITHUB_REPOSITORY}" --jq '.private')"
owner_type="$(gh api "repos/${GITHUB_REPOSITORY}" --jq -r '.owner.type')"
private="$(gh api "repos/${GITHUB_REPOSITORY}" --jq -r '.private')"
if [[ "$owner_type" == "User" && "$private" == "true" ]]; then
echo "enabled=false" >> "$GITHUB_OUTPUT"
echo "Skipping registry attestation: unavailable for private repositories on personal accounts."

View File

@ -1,42 +1,11 @@
import { mkdir, mkdtemp, realpath, writeFile } from "node:fs/promises";
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type * as db from "@rakazo/db";
import { SpaceLimitError } from "@rakazo/db";
import { afterEach, describe, expect, it, vi } from "vitest";
import { getScanStatus, startOnboardingScan } from "./onboarding-scan.js";
const createSpaceForMember = vi.hoisted(() => vi.fn());
vi.mock("@rakazo/db", async (original) => ({
...(await original<typeof db>()),
createSpaceForMember,
}));
const owner = { userId: "user-owner", spaceId: "space-owner", isDeploymentOwner: true };
const member = { userId: "user-member", spaceId: "space-member", isDeploymentOwner: false };
type SpaceUpdate = { where: { id: string }; data: { workspacePath: string } };
/** The spaces the scan reads and links; new ones come from `createSpaceForMember`. */
function fakeDatabase({ spaceLimit = Number.POSITIVE_INFINITY } = {}) {
const spaces: Array<{ id: string; name: string; workspacePath: string | null }> = [];
createSpaceForMember.mockImplementation(async (_prisma: unknown, input: { name: string }) => {
if (spaces.length >= spaceLimit) throw new SpaceLimitError();
const space = { id: `space-${spaces.length + 1}`, name: input.name, workspacePath: null };
spaces.push(space);
return { id: space.id, name: space.name };
});
const prisma = {
space: {
findMany: async () => spaces.filter((space) => space.workspacePath),
update: async ({ where, data }: SpaceUpdate) => {
const space = spaces.find((candidate) => candidate.id === where.id);
if (space) space.workspacePath = data.workspacePath;
},
},
} as unknown as db.PrismaClient;
return { prisma, spaces };
}
const owner = { userId: "user-owner", isDeploymentOwner: true };
const member = { userId: "user-member", isDeploymentOwner: false };
async function scanHomeWithOneFile(): Promise<string> {
const home = await mkdtemp(join(tmpdir(), "evoo-scan-"));
@ -45,12 +14,6 @@ async function scanHomeWithOneFile(): Promise<string> {
return home;
}
async function scanHomeWithRepos(...repos: string[]): Promise<string> {
const home = await scanHomeWithOneFile();
for (const repo of repos) await mkdir(join(home, repo, ".git"), { recursive: true });
return home;
}
async function completed(scanId: string, userId: string) {
return await vi.waitFor(
async () => {
@ -62,21 +25,13 @@ async function completed(scanId: string, userId: string) {
);
}
/** The emulated runtime that the shared test setup pins keeps the scan simulated. */
function allowHostScan() {
vi.stubEnv("AGENT_RUNTIME", "");
vi.stubEnv("SANDBOX_PROVIDER", "");
}
describe("onboarding scan", () => {
afterEach(() => {
vi.unstubAllEnvs();
createSpaceForMember.mockReset();
});
it("hides a scan from every account except the one that started it", async () => {
const { prisma } = fakeDatabase();
const { scanId } = await startOnboardingScan({ actor: member }, prisma);
const { scanId } = await startOnboardingScan({ actor: member }, {});
await expect(getScanStatus(scanId, owner.userId)).rejects.toThrow(/Scan not found/);
await expect(getScanStatus(scanId, member.userId)).resolves.toMatchObject({
status: expect.any(String),
@ -85,77 +40,25 @@ describe("onboarding scan", () => {
});
it("never reads the host without a configured scan home", async () => {
const { prisma } = fakeDatabase();
const { scanId } = await startOnboardingScan({ actor: owner }, prisma);
const { scanId } = await startOnboardingScan({ actor: owner }, {});
const status = await completed(scanId, owner.userId);
expect(status.status).toBe("complete");
expect(status.result).toMatchObject({ localRepos: [], fileSummary: { totalFiles: 0 } });
});
it("lets only the deployment owner discover the configured scan home", async () => {
allowHostScan();
const scanHome = await scanHomeWithRepos("Projects/app");
const { prisma } = fakeDatabase();
// The shared test setup pins the emulated runtime, which also keeps the scan simulated.
vi.stubEnv("AGENT_RUNTIME", "");
vi.stubEnv("SANDBOX_PROVIDER", "");
const scanHome = await scanHomeWithOneFile();
const asMember = await startOnboardingScan({ actor: member, scanHome }, prisma);
const asMember = await startOnboardingScan({ actor: member, scanHome }, {});
const memberStatus = await completed(asMember.scanId, member.userId);
expect(memberStatus.result).toMatchObject({ fileSummary: { totalFiles: 0 } });
expect(createSpaceForMember).not.toHaveBeenCalled();
const asOwner = await startOnboardingScan({ actor: owner, scanHome }, prisma);
const asOwner = await startOnboardingScan({ actor: owner, scanHome }, {});
const ownerStatus = await completed(asOwner.scanId, owner.userId);
expect(ownerStatus.status).toBe("complete");
expect(ownerStatus.result).toMatchObject({ fileSummary: { totalFiles: 1 } });
});
it("links each repository of the deployment owner to one project of its own", async () => {
allowHostScan();
const scanHome = await scanHomeWithRepos("Projects/app", ".config/nvim");
const app = await realpath(join(scanHome, "Projects", "app"));
const { prisma, spaces } = fakeDatabase();
const first = await startOnboardingScan({ actor: owner, scanHome }, prisma);
const firstStatus = await completed(first.scanId, owner.userId);
expect(firstStatus.status).toBe("complete");
expect(firstStatus.result).toMatchObject({
localRepos: [{ name: "app" }],
createdProjects: [{ spaceId: "space-1", name: "app", path: app }],
});
expect(createSpaceForMember).toHaveBeenCalledWith(prisma, {
currentSpaceId: owner.spaceId,
userId: owner.userId,
name: "app",
});
expect(spaces).toEqual([{ id: "space-1", name: "app", workspacePath: app }]);
// Scanning again finds the linked project instead of creating a second one.
const again = await startOnboardingScan({ actor: owner, scanHome }, prisma);
const againStatus = await completed(again.scanId, owner.userId);
expect(againStatus.result).toMatchObject({ createdProjects: [] });
expect(createSpaceForMember).toHaveBeenCalledTimes(1);
});
it("joins a running scan instead of linking its repositories twice", async () => {
allowHostScan();
const scanHome = await scanHomeWithRepos("Projects/app");
const { prisma, spaces } = fakeDatabase();
const first = await startOnboardingScan({ actor: owner, scanHome }, prisma);
const second = await startOnboardingScan({ actor: owner, scanHome }, prisma);
expect(second.scanId).toBe(first.scanId);
await completed(first.scanId, owner.userId);
expect(spaces).toHaveLength(1);
});
it("stops creating projects at the space limit and still completes", async () => {
allowHostScan();
const scanHome = await scanHomeWithRepos("Projects/a", "Projects/b", "Projects/c");
const { prisma } = fakeDatabase({ spaceLimit: 1 });
const { scanId } = await startOnboardingScan({ actor: owner, scanHome }, prisma);
const status = await completed(scanId, owner.userId);
expect(status.status).toBe("complete");
expect(status.result).toMatchObject({ createdProjects: [{ spaceId: "space-1" }] });
expect(createSpaceForMember).toHaveBeenCalledTimes(2);
});
});

View File

@ -2,16 +2,8 @@
// Runs synchronously on the host machine (no sandbox needed).
import { spawn } from "node:child_process";
import { access, readdir, stat } from "node:fs/promises";
import { basename, dirname, join } from "node:path";
import { dirname, join } from "node:path";
import type { Actor } from "@rakazo/contracts";
import type { PrismaClient } from "@rakazo/db";
import {
createSpaceForMember,
InvalidSpaceNameError,
resolveProjectWorkspaceDirectory,
SpaceLimitError,
wireProjectWorkspace,
} from "@rakazo/db";
// ---------------------------------------------------------------------------
// Types
@ -62,9 +54,10 @@ interface ProfileSummary {
}
interface CreatedProject {
spaceId: string;
id: string;
name: string;
path: string;
spaceId: string;
description: string;
}
interface ScanRecord {
@ -122,10 +115,8 @@ async function runSimulatedScan(record: ScanRecord) {
record.completedAt = new Date();
}
type ScanActor = Pick<Actor, "userId" | "spaceId" | "isDeploymentOwner">;
export interface OnboardingScanRequest {
actor: ScanActor;
actor: Pick<Actor, "userId" | "isDeploymentOwner">;
botId?: string;
/** Home-style directory to discover. Unset keeps the scan off the host filesystem. */
scanHome?: string;
@ -133,12 +124,8 @@ export interface OnboardingScanRequest {
export async function startOnboardingScan(
{ actor, botId, scanHome }: OnboardingScanRequest,
prisma: PrismaClient,
prisma: any,
): Promise<{ scanId: string }> {
// A repeated start (a remount, a retry) joins the running scan, so no repository is linked twice.
for (const running of scanRegistry.values()) {
if (running.ownerUserId === actor.userId && !running.completedAt) return { scanId: running.id };
}
const scanId = `scan_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
const record: ScanRecord = {
id: scanId,
@ -167,7 +154,7 @@ export async function startOnboardingScan(
await runSimulatedScan(record);
return;
}
const result = await executeFullScan(scanHome, actor, prisma);
const result = await executeFullScan(scanHome, botId ?? "", prisma);
record.status = "complete";
record.progress = 100;
record.message = "Scan complete — profile ready";
@ -202,11 +189,7 @@ export async function getScanStatus(
// Full scan pipeline
// ---------------------------------------------------------------------------
async function executeFullScan(
homeDir: string,
actor: ScanActor,
prisma: PrismaClient,
): Promise<ScanResult> {
async function executeFullScan(homeDir: string, _botId: string, prisma: any): Promise<ScanResult> {
// Phase 1: Local file scan
const [fileSummary, projectFolders] = await scanFiles(homeDir);
@ -217,8 +200,8 @@ async function executeFullScan(
// Phase 4: Profile compilation from findings
const profileSummary = compileProfile(localRepos, fileSummary, projectFolders);
// Phase 6: Projects for discovered repositories
const createdProjects = await createProjects(localRepos, actor, prisma);
// Phase 6: Project folder creation
const createdProjects = await createProjectFolders(localRepos, prisma);
// Phase 7: Sub-agent role creation
const subAgentRoles = generateSubAgentRoles(profileSummary, localRepos);
@ -430,9 +413,9 @@ async function findGitDirectories(dir: string, depth: number = 0): Promise<strin
if (entry.name === ".git") {
results.push(dirname(fullPath));
} else if (
// Dot folders hold tools and caches (shell and editor plugins), not the person's projects.
!entry.name.startsWith(".") &&
entry.name !== "node_modules" &&
entry.name !== ".git" &&
entry.name !== ".venv" &&
entry.name !== "dist"
) {
results.push(...(await findGitDirectories(fullPath, depth + 1)));
@ -639,44 +622,38 @@ function generateBriefing(
}
// ---------------------------------------------------------------------------
// Phase 5: Projects for discovered repositories
// Phase 5: Project folder creation
// ---------------------------------------------------------------------------
/** Give each discovered repository a project linked to its folder, unless one already links it. */
async function createProjects(
repos: RepoInfo[],
actor: ScanActor,
prisma: PrismaClient,
): Promise<CreatedProject[]> {
const linked = await prisma.space.findMany({
where: { workspacePath: { not: null }, memberships: { some: { userId: actor.userId } } },
select: { workspacePath: true },
});
const linkedFolders = new Set(linked.map((space) => space.workspacePath));
async function createProjectFolders(repos: RepoInfo[], prisma: any): Promise<CreatedProject[]> {
const created: CreatedProject[] = [];
for (const repo of repos) {
// Compare and link the real folder, so the next scan recognizes the projects of this one.
// Checking it before the project exists means a folder that cannot be linked leaves none.
const folder = await resolveProjectWorkspaceDirectory(repo.path).catch(() => null);
if (!folder || linkedFolders.has(folder)) continue;
let space: { id: string; name: string };
try {
space = await createSpaceForMember(prisma, {
currentSpaceId: actor.spaceId,
userId: actor.userId,
name: basename(folder),
// Check if space already exists
const existing = await prisma.space.findFirst({ where: { title: repo.name } });
if (existing) {
created.push({
id: existing.id,
name: repo.name,
spaceId: existing.id,
description: repo.remote || repo.path,
});
continue;
}
// Create new space
const space = await prisma.space.create({
data: { title: repo.name, path: repo.path },
});
} catch (error) {
if (error instanceof SpaceLimitError) break;
if (error instanceof InvalidSpaceNameError) continue;
throw error;
created.push({
id: space.id,
name: repo.name,
spaceId: space.id,
description: repo.remote || repo.path,
});
} catch {
// Space might already exist or DB unavailable — skip
}
const workspace = await wireProjectWorkspace(prisma, {
spaceId: space.id,
workspacePath: folder,
});
created.push({ spaceId: space.id, name: space.name, path: workspace.path });
}
return created;

View File

@ -137,7 +137,6 @@ import {
releaseSpaceDeletionClaim,
renewSpaceDeletionClaim,
requireMembership,
resolveProjectWorkspaceDirectory,
SPACE_DELETION_CLAIM_TIMEOUT_MS,
SpaceDeletionInProgressError,
SpaceLimitError,
@ -570,22 +569,6 @@ export function createRouter(deps: RouterDeps) {
spaceNavigationDto(deps, context.actor, repos, groupRepos),
),
create: authed.spaces.create.handler(async ({ context, input }) => {
// A linked folder lives on the host, and the host is the deployment owner's machine.
if (input.workspacePath && !context.actor.isDeploymentOwner) {
throw new ORPCError("FORBIDDEN");
}
// Check the folder before the space exists, so a bad path leaves no space behind.
let workspacePath: string | null = null;
if (input.workspacePath) {
try {
workspacePath = await resolveProjectWorkspaceDirectory(input.workspacePath);
} catch (error) {
if (error instanceof InvalidProjectWorkspacePathError) {
throw new ORPCError("BAD_REQUEST", { message: error.message });
}
throw error;
}
}
let space: { id: string; name: string };
try {
space = await createSpaceForMember(deps.prisma, {
@ -599,9 +582,20 @@ export function createRouter(deps: RouterDeps) {
}
throw error;
}
const workspace = workspacePath
? await wireProjectWorkspace(deps.prisma, { spaceId: space.id, workspacePath })
: null;
let workspace = null;
if (input.workspacePath) {
try {
workspace = await wireProjectWorkspace(deps.prisma, {
spaceId: space.id,
workspacePath: input.workspacePath,
});
} catch (error) {
if (error instanceof InvalidProjectWorkspacePathError) {
throw new ORPCError("BAD_REQUEST", { message: error.message });
}
throw error;
}
}
return {
id: space.id,
name: space.name,
@ -2236,7 +2230,10 @@ export function createRouter(deps: RouterDeps) {
}
}
if (isProjectWorkspacePath(storedPath)) {
const workspaceRoot = await requireProjectWorkspaceRoot(deps.prisma, context.actor);
const workspaceRoot = await requireProjectWorkspaceRoot(
deps.prisma,
context.actor.spaceId,
);
const entries = await listProjectWorkspaceFiles(workspaceRoot, storedPath);
return entries.map((entry) => ({
...entry,
@ -2284,7 +2281,10 @@ export function createRouter(deps: RouterDeps) {
}
}
if (isProjectWorkspacePath(storedPath)) {
const workspaceRoot = await requireProjectWorkspaceRoot(deps.prisma, context.actor);
const workspaceRoot = await requireProjectWorkspaceRoot(
deps.prisma,
context.actor.spaceId,
);
const bytes = await readProjectWorkspaceFile(workspaceRoot, storedPath, {
maxBytes: MAX_COMPUTER_TEXT_FILE_BYTES,
});
@ -4894,13 +4894,10 @@ function mapUpdaterError(error: unknown): never {
async function requireProjectWorkspaceRoot(
prisma: Pick<PrismaClient, "space">,
actor: Actor,
spaceId: string,
): Promise<string> {
// Only the deployment owner may link a host folder, so only they may browse one. This also
// closes folders that other members linked before linking was limited to the owner.
if (!actor.isDeploymentOwner) throw new ORPCError("FORBIDDEN");
const space = await prisma.space.findUnique({
where: { id: actor.spaceId },
where: { id: spaceId },
select: { workspacePath: true },
});
if (!space?.workspacePath) {
@ -5012,8 +5009,7 @@ async function spaceNavigationDto(
memberships.length > 1 &&
!spacesWithContent.has(membership.spaceId) &&
(membership.space.deletingAt === null || membership.space.deletingAt < staleClaimBefore),
// A linked folder is on the owner's host; other members see neither its path nor remote.
workspace: actor.isDeploymentOwner ? projectWorkspaceDto(membership.space) : null,
workspace: projectWorkspaceDto(membership.space),
bots: spaceBots.map((bot) => ({
id: bot.id,
spaceId: bot.spaceId,

View File

@ -1,155 +0,0 @@
import http from "node:http";
import path from "node:path";
import { _electron as electron, expect, test } from "@playwright/test";
import type { RakazoDesktop, RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
const fixture = `<!doctype html>
<html lang="en">
<head><meta charset="utf-8"><title>Rakazo desktop browser</title></head>
<body><main>App with a browser panel</main></body>
</html>`;
type Bridge = NonNullable<RakazoDesktop["browserPreview"]>;
test("the in-app browser is a native view the app window lays over its panel", async () => {
const pages = http.createServer((req, res) => {
res.setHeader("content-type", "text/html");
if (req.url === "/second") {
res.end("<!doctype html><title>Second page</title><p>second</p>");
return;
}
res.end(`<!doctype html><title>First page</title>
<a id="next" href="/second">next</a>
<a id="popup" href="/second" target="_blank">popup</a>
<a id="file" href="file:///etc/hosts">file</a>`);
});
await new Promise<void>((resolve) => pages.listen(0, "127.0.0.1", resolve));
const address = pages.address();
if (!address || typeof address === "string") throw new Error("No fixture server");
const origin = `http://127.0.0.1:${address.port}`;
const app = await electron.launch({
args: ["."],
cwd: path.resolve(import.meta.dirname, ".."),
env: {
...process.env,
RAKAZO_WEB_URL: `data:text/html;charset=utf-8,${encodeURIComponent(fixture)}`,
},
});
try {
const page = await app.firstWindow();
await expect(page.getByText("App with a browser panel")).toBeVisible();
const call = <K extends keyof Bridge>(method: K, ...args: unknown[]) =>
page.evaluate(
([name, values]) => {
const bridge = (window as typeof window & { rakazoDesktop?: RakazoDesktop }).rakazoDesktop
?.browserPreview as unknown as Record<
string,
(...input: unknown[]) => Promise<unknown>
>;
return bridge[name as string]!(...(values as unknown[]));
},
[method, args] as const,
);
const state = () => call("state") as Promise<RakazoDesktopBrowserPreviewState>;
const view = () =>
app.evaluate(({ BrowserWindow }) => {
const win = BrowserWindow.getAllWindows()[0]!;
const child = win.contentView.children[0] as
| (Electron.View & { webContents?: Electron.WebContents })
| undefined;
return child
? {
count: win.contentView.children.length,
visible: child.getVisible(),
bounds: child.getBounds(),
url: child.webContents?.getURL(),
partition: child.webContents?.session === win.webContents.session,
sandbox: child.webContents?.getLastWebPreferences()?.sandbox,
nodeIntegration: child.webContents?.getLastWebPreferences()?.nodeIntegration,
}
: null;
});
// Pages load in a native view in their own session, never the app's.
await call("navigate", `${origin}/`);
await expect.poll(async () => (await state()).title).toBe("First page");
expect(await view()).toMatchObject({
count: 1,
visible: false,
url: `${origin}/`,
partition: false,
sandbox: true,
nodeIntegration: false,
});
await call("show", { x: 400, y: 50, width: 300, height: 200 });
expect(await view()).toMatchObject({
visible: true,
bounds: { x: 400, y: 50, width: 300, height: 200 },
});
// Renderer bounds outside the window are clamped to it.
await call("show", { x: -10, y: -10, width: 100_000, height: 100_000 });
const [width, height] = await app.evaluate(({ BrowserWindow }) =>
BrowserWindow.getAllWindows()[0]!.getContentSize(),
);
expect((await view())?.bounds).toEqual({ x: 0, y: 0, width, height });
const contents = () =>
app.evaluate(({ BrowserWindow }, script) => {
const win = BrowserWindow.getAllWindows()[0]!;
const child = win.contentView.children[0] as Electron.View & {
webContents: Electron.WebContents;
};
return child.webContents.executeJavaScript(script, true);
}, "document.getElementById('next').click()");
await contents();
await expect.poll(async () => (await state()).title).toBe("Second page");
expect(await state()).toMatchObject({ canGoBack: true, canGoForward: false, error: null });
await call("back");
await expect.poll(async () => (await state()).title).toBe("First page");
expect(await state()).toMatchObject({ canGoForward: true });
await call("forward");
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
// New-window links stay in the one view; file links never load.
await call("back");
await expect.poll(async () => (await state()).title).toBe("First page");
await app.evaluate(({ BrowserWindow }) => {
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
webContents: Electron.WebContents;
};
return child.webContents.executeJavaScript("document.getElementById('popup').click()", true);
});
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
expect(await app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().length)).toBe(1);
await call("back");
await expect.poll(async () => (await state()).title).toBe("First page");
await app.evaluate(({ BrowserWindow }) => {
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
webContents: Electron.WebContents;
};
return child.webContents.executeJavaScript("document.getElementById('file').click()", true);
});
await page.waitForTimeout(300);
expect((await state()).url).toBe(`${origin}/`);
await expect(call("navigate", "file:///etc/hosts")).rejects.toThrow(/http/);
// A dev server that is not running yet shows as an error, then recovers on reload.
pages.close();
await new Promise((resolve) => setTimeout(resolve, 100));
await call("navigate", `${origin}/gone`);
await expect.poll(async () => (await state()).error).toMatch(/ERR_CONNECTION_REFUSED/);
await call("hide");
expect((await view())?.visible).toBe(false);
} finally {
await app.close();
pages.closeAllConnections();
pages.close();
}
});

View File

@ -42,7 +42,6 @@ test("launches with a narrow preload bridge and an isolated renderer", async ()
});
expect(renderer.bridgeKeys).toEqual([
"browserPreview",
"files",
"localSettings",
"oauth",

View File

@ -1,96 +0,0 @@
import { describe, expect, it } from "vitest";
import {
previewBounds,
previewLoadError,
previewShortcut,
previewUrl,
} from "./browser-preview-policy.js";
describe("previewUrl", () => {
it("accepts only http(s) without credentials", () => {
expect(previewUrl("http://localhost:5173")).toBe("http://localhost:5173/");
expect(previewUrl("https://example.com/a?b=1")).toBe("https://example.com/a?b=1");
for (const value of [
"file:///etc/hosts",
"javascript:alert(1)",
"chrome://settings",
"https://user:fake-password@example.com",
"not a url",
42,
`https://example.com/${"a".repeat(9000)}`,
]) {
expect(previewUrl(value)).toBeNull();
}
});
});
describe("previewBounds", () => {
const content = { width: 1200, height: 800 };
it("rounds and applies the app zoom factor", () => {
expect(previewBounds({ x: 800.4, y: 40.6, width: 399.5, height: 700 }, content)).toEqual({
x: 800,
y: 41,
width: 400,
height: 700,
});
expect(previewBounds({ x: 100, y: 10, width: 200, height: 100 }, content, 1.25)).toEqual({
x: 125,
y: 13,
width: 250,
height: 125,
});
});
it("clamps to the window and rejects empty or malformed rectangles", () => {
expect(previewBounds({ x: -50, y: -10, width: 5000, height: 5000 }, content)).toEqual({
x: 0,
y: 0,
width: 1200,
height: 800,
});
expect(previewBounds({ x: 1300, y: 0, width: 100, height: 100 }, content)).toBeNull();
expect(previewBounds({ x: 0, y: 0, width: 0, height: 100 }, content)).toBeNull();
expect(previewBounds({ x: "1", y: 0, width: 1, height: 1 }, content)).toBeNull();
expect(previewBounds({ x: Number.NaN, y: 0, width: 1, height: 1 }, content)).toBeNull();
expect(previewBounds(null, content)).toBeNull();
});
});
describe("previewShortcut", () => {
const key = (overrides: Partial<Parameters<typeof previewShortcut>[0]>) => ({
type: "keyDown",
key: "r",
control: false,
meta: false,
alt: false,
shift: false,
...overrides,
});
it("maps platform browser shortcuts", () => {
expect(previewShortcut(key({ meta: true }), "darwin")).toBe("reload");
expect(previewShortcut(key({ meta: true, shift: true, key: "R" }), "darwin")).toBe(
"hard-reload",
);
expect(previewShortcut(key({ control: true }), "linux")).toBe("reload");
expect(previewShortcut(key({ meta: true, key: "[" }), "darwin")).toBe("back");
expect(previewShortcut(key({ alt: true, key: "ArrowRight" }), "win32")).toBe("forward");
expect(previewShortcut(key({ key: "F12" }), "linux")).toBe("devtools");
expect(previewShortcut(key({ key: "F5" }), "win32")).toBe("reload");
});
it("leaves typing alone", () => {
expect(previewShortcut(key({}), "darwin")).toBeNull();
expect(previewShortcut(key({ control: true }), "darwin")).toBeNull();
expect(previewShortcut(key({ type: "keyUp", meta: true }), "darwin")).toBeNull();
expect(previewShortcut(key({ key: "ArrowLeft" }), "linux")).toBeNull();
});
});
describe("previewLoadError", () => {
it("ignores superseded navigations", () => {
expect(previewLoadError(-3, "ERR_ABORTED")).toBeNull();
expect(previewLoadError(-102, "ERR_CONNECTION_REFUSED")).toBe("ERR_CONNECTION_REFUSED");
});
});

View File

@ -1,76 +0,0 @@
import type { RakazoDesktopBrowserBounds } from "@rakazo/contracts";
/** Only plain http(s) pages load in the in-app browser; no files, schemes or URL credentials. */
export function previewUrl(value: unknown): string | null {
if (typeof value !== "string" || value.length > 8192) return null;
try {
const url = new URL(value);
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
if (url.username || url.password) return null;
return url.toString();
} catch {
return null;
}
}
/**
* Renderer-reported CSS-pixel bounds, converted to window DIPs and clamped to the content area
* so a compromised renderer cannot place the view outside the panel it reserved.
*/
export function previewBounds(
value: unknown,
content: { width: number; height: number },
zoomFactor = 1,
): RakazoDesktopBrowserBounds | null {
if (!value || typeof value !== "object") return null;
const raw = value as Record<string, unknown>;
const numbers = [raw.x, raw.y, raw.width, raw.height];
if (!numbers.every((n) => typeof n === "number" && Number.isFinite(n))) return null;
const scale = Number.isFinite(zoomFactor) && zoomFactor > 0 ? zoomFactor : 1;
const [x, y, width, height] = (numbers as number[]).map((n) => Math.round(n * scale)) as [
number,
number,
number,
number,
];
const left = Math.min(Math.max(0, x), content.width);
const top = Math.min(Math.max(0, y), content.height);
const right = Math.min(Math.max(left, x + width), content.width);
const bottom = Math.min(Math.max(top, y + height), content.height);
if (right - left < 1 || bottom - top < 1) return null;
return { x: left, y: top, width: right - left, height: bottom - top };
}
export type PreviewShortcut = "reload" | "hard-reload" | "back" | "forward" | "devtools";
/** Browser keyboard shortcuts while focus is inside the page. */
export function previewShortcut(
input: {
type: string;
key: string;
control: boolean;
meta: boolean;
alt: boolean;
shift: boolean;
},
platform: string,
): PreviewShortcut | null {
if (input.type !== "keyDown") return null;
const mod = platform === "darwin" ? input.meta : input.control;
const key = input.key.length === 1 ? input.key.toLowerCase() : input.key;
if (key === "F5") return input.shift ? "hard-reload" : "reload";
if (key === "F12") return "devtools";
if (mod && key === "r") return input.shift ? "hard-reload" : "reload";
if (mod && input.alt && key === "i") return "devtools";
if (platform === "darwin" ? input.meta && !input.alt : input.alt && !input.control) {
if (key === (platform === "darwin" ? "[" : "ArrowLeft")) return "back";
if (key === (platform === "darwin" ? "]" : "ArrowRight")) return "forward";
}
return null;
}
/** Load failures worth showing; an aborted load (-3) is a superseded navigation, not an error. */
export function previewLoadError(errorCode: number, description: string): string | null {
if (errorCode === -3 || errorCode === 0) return null;
return description || `Load failed (${errorCode})`;
}

View File

@ -1,189 +0,0 @@
import type { RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
import { type BrowserWindow, WebContentsView } from "electron";
import {
previewBounds,
previewLoadError,
previewShortcut,
previewUrl,
} from "./browser-preview-policy.js";
/** Pages share one persistent profile, separate from the app's own session and cookies. */
const PREVIEW_PARTITION = "persist:rakazo-browser";
export const PREVIEW_STATE_CHANNEL = "desktop.browserPreview.state";
/**
* The in-app browser's native page: one WebContentsView laid over the panel the app window
* reserves. It belongs to one app window at a time and is torn down with it.
*/
export class BrowserPreviewController {
private host: BrowserWindow | null = null;
private view: WebContentsView | null = null;
private error: string | null = null;
state(win: BrowserWindow): RakazoDesktopBrowserPreviewState {
const contents = this.host === win ? this.view?.webContents : undefined;
if (!contents || contents.isDestroyed()) {
return {
url: "",
title: "",
loading: false,
canGoBack: false,
canGoForward: false,
error: null,
};
}
const url = contents.getURL();
return {
url,
title: contents.getTitle() === url ? "" : contents.getTitle(),
loading: contents.isLoading(),
canGoBack: contents.navigationHistory.canGoBack(),
canGoForward: contents.navigationHistory.canGoForward(),
error: this.error,
};
}
show(win: BrowserWindow, rawBounds: unknown) {
const view = this.viewFor(win);
const [width, height] = win.getContentSize();
const bounds = previewBounds(
rawBounds,
{ width: width ?? 0, height: height ?? 0 },
win.webContents.getZoomFactor(),
);
if (!bounds) {
view.setVisible(false);
return;
}
view.setBounds(bounds);
view.setVisible(true);
}
hide(win: BrowserWindow) {
if (this.host === win) this.view?.setVisible(false);
}
async navigate(win: BrowserWindow, rawUrl: unknown) {
const url = previewUrl(rawUrl);
if (!url) throw new Error("Only http(s) pages can open here.");
this.error = null;
// Failures surface through did-fail-load as page state, not as a rejected call.
await this.viewFor(win)
.webContents.loadURL(url)
.catch(() => undefined);
}
back(win: BrowserWindow) {
const history = this.contentsFor(win)?.navigationHistory;
if (history?.canGoBack()) history.goBack();
}
forward(win: BrowserWindow) {
const history = this.contentsFor(win)?.navigationHistory;
if (history?.canGoForward()) history.goForward();
}
reload(win: BrowserWindow, ignoreCache = false) {
const contents = this.contentsFor(win);
if (!contents?.getURL()) return;
this.error = null;
if (ignoreCache) contents.reloadIgnoringCache();
else contents.reload();
}
stop(win: BrowserWindow) {
this.contentsFor(win)?.stop();
}
openDevTools(win: BrowserWindow) {
const contents = this.contentsFor(win);
if (contents?.getURL()) contents.openDevTools({ mode: "detach" });
}
private contentsFor(win: BrowserWindow) {
const contents = this.host === win ? this.view?.webContents : undefined;
return contents && !contents.isDestroyed() ? contents : undefined;
}
private viewFor(win: BrowserWindow): WebContentsView {
if (this.host === win && this.view && !this.view.webContents.isDestroyed()) return this.view;
this.destroy();
const view = new WebContentsView({
webPreferences: {
partition: PREVIEW_PARTITION,
sandbox: true,
contextIsolation: true,
nodeIntegration: false,
webSecurity: true,
spellcheck: false,
},
});
view.setVisible(false);
const contents = view.webContents;
const publish = () => {
if (!win.isDestroyed()) win.webContents.send(PREVIEW_STATE_CHANNEL, this.state(win));
};
// One tab: links that want a new window open here instead.
contents.setWindowOpenHandler(({ url }) => {
const target = previewUrl(url);
if (target) void contents.loadURL(target).catch(() => undefined);
return { action: "deny" };
});
contents.on("will-navigate", (event, url) => {
if (!previewUrl(url)) event.preventDefault();
});
contents.on("will-redirect", (event, url) => {
if (!previewUrl(url)) event.preventDefault();
});
contents.on("did-start-loading", publish);
contents.on("did-stop-loading", publish);
contents.on("did-navigate", () => {
this.error = null;
publish();
});
contents.on("did-navigate-in-page", publish);
contents.on("page-title-updated", publish);
contents.on("did-fail-load", (_event, code, description, _url, isMainFrame) => {
if (!isMainFrame) return;
const error = previewLoadError(code, description);
if (error === null) return;
this.error = error;
publish();
});
contents.on("render-process-gone", (_event, details) => {
this.error = details.reason === "clean-exit" ? null : "The page stopped responding.";
publish();
});
contents.on("before-input-event", (event, input) => {
const shortcut = previewShortcut(input, process.platform);
if (!shortcut) return;
event.preventDefault();
if (shortcut === "reload" || shortcut === "hard-reload") {
this.reload(win, shortcut === "hard-reload");
} else if (shortcut === "back") {
this.back(win);
} else if (shortcut === "forward") {
this.forward(win);
} else {
this.openDevTools(win);
}
});
win.contentView.addChildView(view);
win.once("closed", () => {
if (this.host === win) this.destroy();
});
this.host = win;
this.view = view;
return view;
}
private destroy() {
const { host, view } = this;
this.host = null;
this.view = null;
this.error = null;
if (!view) return;
if (host && !host.isDestroyed()) host.contentView.removeChildView(view);
if (!view.webContents.isDestroyed()) view.webContents.close();
}
}

View File

@ -21,7 +21,6 @@ import {
LAUNCH_CHECK_DELAY_MS,
} from "./auto-update.js";
import { openBrowserAuth } from "./browser-auth.js";
import { BrowserPreviewController } from "./browser-preview.js";
import { DOCKER_INSTALL_LINKS, isDesktopSetupLink, runDocker } from "./docker-cli.js";
import { requestLocalSettings } from "./local-settings.js";
import {
@ -1131,35 +1130,6 @@ app.whenReady().then(async () => {
},
);
ipcMain.handle("desktop.platform", () => process.platform);
const browserPreview = new BrowserPreviewController();
// The in-app browser answers only the app window's own top frame, never a page inside it.
const previewHost = (event: Electron.IpcMainInvokeEvent) => {
if (!fromMainWindow(event) || event.senderFrame !== event.sender.mainFrame) {
throw new Error("The in-app browser is not available here.");
}
return mainWindow!;
};
ipcMain.handle("desktop.browserPreview.state", (event) =>
browserPreview.state(previewHost(event)),
);
ipcMain.handle("desktop.browserPreview.show", (event, bounds: unknown) =>
browserPreview.show(previewHost(event), bounds),
);
ipcMain.handle("desktop.browserPreview.hide", (event) => browserPreview.hide(previewHost(event)));
ipcMain.handle("desktop.browserPreview.navigate", (event, url: unknown) =>
browserPreview.navigate(previewHost(event), url),
);
ipcMain.handle("desktop.browserPreview.back", (event) => browserPreview.back(previewHost(event)));
ipcMain.handle("desktop.browserPreview.forward", (event) =>
browserPreview.forward(previewHost(event)),
);
ipcMain.handle("desktop.browserPreview.reload", (event) =>
browserPreview.reload(previewHost(event)),
);
ipcMain.handle("desktop.browserPreview.stop", (event) => browserPreview.stop(previewHost(event)));
ipcMain.handle("desktop.browserPreview.openDevTools", (event) =>
browserPreview.openDevTools(previewHost(event)),
);
ipcMain.handle("desktop.files.pickDirectory", async (event) => {
if (!fromMainWindow(event)) return null;
const result = await dialog.showOpenDialog(windowFrom(event)!, {

View File

@ -21,22 +21,6 @@ contextBridge.exposeInMainWorld("rakazoDesktop", {
download: () => ipcRenderer.invoke("desktop.update.download"),
install: () => ipcRenderer.invoke("desktop.update.install"),
},
browserPreview: {
state: () => ipcRenderer.invoke("desktop.browserPreview.state"),
show: (bounds) => ipcRenderer.invoke("desktop.browserPreview.show", bounds),
hide: () => ipcRenderer.invoke("desktop.browserPreview.hide"),
navigate: (url) => ipcRenderer.invoke("desktop.browserPreview.navigate", url),
back: () => ipcRenderer.invoke("desktop.browserPreview.back"),
forward: () => ipcRenderer.invoke("desktop.browserPreview.forward"),
reload: () => ipcRenderer.invoke("desktop.browserPreview.reload"),
stop: () => ipcRenderer.invoke("desktop.browserPreview.stop"),
openDevTools: () => ipcRenderer.invoke("desktop.browserPreview.openDevTools"),
onState: (listener) => {
const handler = (_event, state) => listener(state);
ipcRenderer.on("desktop.browserPreview.state", handler);
return () => ipcRenderer.off("desktop.browserPreview.state", handler);
},
},
oauth: {
open: (url) => ipcRenderer.invoke("desktop.oauth.open", url),
cancel: (url) => ipcRenderer.invoke("desktop.oauth.cancel", url),

View File

@ -32,7 +32,6 @@ describe("desktop preload bridge", () => {
expect(globalName).toBe("rakazoDesktop");
expect(bridge.platform).toBe("linux");
expect(Object.keys(bridge).sort()).toEqual([
"browserPreview",
"files",
"localSettings",
"oauth",
@ -47,18 +46,6 @@ describe("desktop preload bridge", () => {
"toggleMaximize",
]);
expect(Object.keys(bridge.update).sort()).toEqual(["check", "download", "install", "state"]);
expect(Object.keys(bridge.browserPreview ?? {}).sort()).toEqual([
"back",
"forward",
"hide",
"navigate",
"onState",
"openDevTools",
"reload",
"show",
"state",
"stop",
]);
await bridge.files?.pickDirectory();
await bridge.oauth.open?.("https://provider.example.com/authorize");
@ -71,8 +58,6 @@ describe("desktop preload bridge", () => {
await bridge.update.check();
await bridge.update.download();
await bridge.update.install();
await bridge.browserPreview?.navigate("http://localhost:5173/");
await bridge.browserPreview?.show({ x: 1, y: 2, width: 3, height: 4 });
expect(invoke.mock.calls.map(([channel]) => channel)).toEqual([
"desktop.files.pickDirectory",
"desktop.oauth.open",
@ -85,8 +70,6 @@ describe("desktop preload bridge", () => {
"desktop.update.check",
"desktop.update.download",
"desktop.update.install",
"desktop.browserPreview.navigate",
"desktop.browserPreview.show",
]);
});
@ -94,7 +77,6 @@ describe("desktop preload bridge", () => {
const { exposeInMainWorld } = runPreload("preload.cjs");
const [, bridge] = exposeInMainWorld.mock.calls[0] as [string, Record<string, unknown>];
expect(Object.keys(bridge).sort()).toEqual([
"browserPreview",
"files",
"localSettings",
"oauth",

View File

@ -1074,7 +1074,7 @@ export function applyMobileThreadEvent(
messages: [...remaining, streaming],
};
}
if (event.type === "agent.tool.completed" || event.type === "thread.computer") {
if (event.type === "agent.tool.completed") {
return { ...prev, cursor: event.seq ?? prev.cursor };
}
if (event.type === "thread.subagent") {

View File

@ -1,53 +0,0 @@
import { expect, test } from "@playwright/test";
import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers";
const SCREEN_URL = "https://screen.example/embed.html";
test("the bot's cursor glides to where it clicks on its live screen", async ({
page,
}, testInfo) => {
await signup(page, `agent-cursor-${Date.now()}@rakazo.test`, "password12", "Cursor Watcher");
await completeOnboarding(page);
const botId = activeBotId(page);
await rpc(page, "computer/boot", { botId });
// Stand in for the noVNC viewer: a 16:10 desktop with a field in the middle.
await page.route("https://screen.example/**", (route) =>
route.fulfill({
contentType: "text/html",
body: `<!doctype html><title>Desktop</title><body style="margin:0;background:#1d1d20;display:grid;place-items:center;height:100vh">
<input style="width:40%;padding:8px;font:16px system-ui" placeholder="Search">`,
}),
);
await page.route("**/rpc/computer/screenUrl", (route) =>
route.fulfill({
contentType: "application/json",
body: JSON.stringify({ json: { url: SCREEN_URL } }),
}),
);
await page.getByTitle("Agent computer").click();
const preview = page.getByTestId("computer-preview");
await expect(preview.locator("iframe")).toHaveAttribute("src", SCREEN_URL);
const cursor = preview.getByTestId("agent-cursor");
await expect(cursor).toHaveCSS("opacity", "0");
// The scripted runtime clicks the screen center (640, 400) and types.
await rpc(page, "threads/send", { botId, text: "use your screen and type cursor-demo" });
await expect(cursor).toHaveCSS("opacity", "1");
await expect(cursor).toHaveAttribute("data-action", /click|type/);
const layer = (await preview.getByTestId("agent-cursor-layer").boundingBox())!;
await expect
.poll(async () => {
const tip = (await cursor.boundingBox())!;
return [
Math.round(tip.x - layer.x - layer.width / 2),
Math.round(tip.y - layer.y - layer.height / 2),
];
})
.toEqual([0, 0]);
await captureScreenshot(page, testInfo, "agent-cursor-on-screen");
// It fades once the bot stops pointing, so it never lingers over the screen.
await expect(cursor).toHaveCSS("opacity", "0", { timeout: 10_000 });
});

View File

@ -1,159 +0,0 @@
import { expect, test } from "@playwright/test";
import { captureScreenshot, completeOnboarding, signup } from "./helpers";
test("the in-app browser previews a page and follows it back to the browser", async ({
page,
}, testInfo) => {
await signup(page, `in-app-browser-${Date.now()}@rakazo.test`, "password12", "Previewer");
await completeOnboarding(page);
// Context-wide so the "Open in browser" tab is served too.
await page.context().route("https://preview.example/**", (route) =>
route.fulfill({
contentType: "text/html",
body: `<!doctype html><title>Preview</title><body style="font:16px system-ui;padding:24px">
<h1>Live preview</h1><p>Rendered inside the app.</p>`,
}),
);
await page.getByRole("button", { name: "Browser" }).click();
const browser = page.getByTestId("in-app-browser");
const address = browser.getByRole("textbox", { name: "Address" });
await expect(address).toBeFocused();
await address.fill("not an address");
await address.press("Enter");
await expect(address).toHaveAttribute("aria-invalid", "true");
await address.fill("preview.example/app");
await address.press("Enter");
await expect(address).toHaveValue("https://preview.example/app");
const frame = browser.getByTestId("in-app-browser-frame");
await expect(frame).toHaveAttribute("src", "https://preview.example/app");
await expect(
page.frameLocator('[data-testid="in-app-browser-frame"]').getByRole("heading"),
).toHaveText("Live preview");
await captureScreenshot(page, testInfo, "in-app-browser");
const popup = page.waitForEvent("popup");
await browser.getByRole("button", { name: "Open in browser" }).click();
expect((await popup).url()).toBe("https://preview.example/app");
await browser.getByRole("button", { name: "Close panel" }).click();
await expect(browser).toHaveCount(0);
// The last address comes back when the browser reopens.
await page.getByRole("button", { name: "Browser" }).click();
await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute(
"src",
"https://preview.example/app",
);
});
test("on desktop the page is a native view glued to the panel", async ({ page }) => {
await page.addInitScript(() => {
const calls: Array<[string, unknown?]> = [];
let listener: ((state: unknown) => void) | undefined;
let state = {
url: "",
title: "",
loading: false,
canGoBack: false,
canGoForward: false,
error: null as string | null,
};
const record = (name: string) => async (value?: unknown) => {
calls.push([name, value]);
};
Object.assign(window, { __browserCalls: calls });
Object.defineProperty(window, "rakazoDesktop", {
value: {
platform: "linux",
window: {
close: async () => {},
minimize: async () => {},
toggleMaximize: async () => {},
state: async () => ({ minimized: false, maximized: false, fullScreen: false }),
},
oauth: { onCallback: () => () => {} },
update: {
state: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
check: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
download: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
install: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
},
browserPreview: {
state: async () => state,
onState: (next: (value: unknown) => void) => {
listener = next;
return () => {
listener = undefined;
};
},
navigate: async (url: string) => {
calls.push(["navigate", url]);
state = { ...state, url, title: "Dev server", loading: false, canGoBack: true };
listener?.(state);
},
show: record("show"),
hide: record("hide"),
back: record("back"),
forward: record("forward"),
reload: record("reload"),
stop: record("stop"),
openDevTools: record("openDevTools"),
},
},
});
});
await signup(page, `native-browser-${Date.now()}@rakazo.test`, "password12", "Desktop");
await completeOnboarding(page);
const calls = () =>
page.evaluate(
() => (window as unknown as { __browserCalls: Array<[string, unknown?]> }).__browserCalls,
);
await page.getByRole("button", { name: "Browser" }).click();
const browser = page.getByTestId("in-app-browser");
await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173");
await browser.getByRole("textbox", { name: "Address" }).press("Enter");
await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]);
await expect(browser.getByRole("button", { name: "Back" })).toBeEnabled();
await expect(browser.getByRole("button", { name: "Forward" })).toBeDisabled();
await expect(browser.getByTestId("in-app-browser-frame")).toHaveCount(0);
// The reserved page area and the native view's bounds stay identical.
const viewport = browser.getByTestId("in-app-browser-viewport");
await expect
.poll(async () => {
const box = (await viewport.boundingBox())!;
const shown = (await calls()).filter(([name]) => name === "show").at(-1)?.[1];
return (
JSON.stringify(shown) ===
JSON.stringify({
x: Math.round(box.x),
y: Math.round(box.y),
width: Math.round(box.width),
height: Math.round(box.height),
})
);
})
.toBe(true);
await browser.getByRole("button", { name: "Back" }).click();
await browser.getByRole("button", { name: "Reload" }).click();
await browser.getByRole("button", { name: "Developer tools" }).click();
await expect
.poll(async () => (await calls()).map(([name]) => name))
.toEqual(expect.arrayContaining(["back", "reload", "openDevTools"]));
// App menus float above the page, so the native view steps aside while one is open.
const before = (await calls()).length;
await page.getByTestId("create-menu-trigger").click();
await expect
.poll(async () => (await calls()).slice(before).map(([name]) => name))
.toContain("hide");
await page.keyboard.press("Escape");
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("show");
await browser.getByRole("button", { name: "Close panel" }).click();
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("hide");
});

View File

@ -50,16 +50,6 @@ test("live page helper preserves identity, masks passwords, and reports partial
expect(snapshot.title).toBe("Fixture");
expect(snapshot.tree).toContain("Readable article content");
expect(JSON.stringify(snapshot)).not.toContain("fake-test-password");
// Element boxes are screen pixels for the visible bot cursor.
expect(snapshot.screen.width).toBeGreaterThan(0);
expect(snapshot.screen.height).toBeGreaterThan(0);
const saveBox = snapshot.elements.find((element: { name: string }) => element.name === "Save")
.box as number[];
const saveRect = (await page.locator("#save").boundingBox())!;
expect(saveBox[2]).toBe(Math.round(saveRect.width));
expect(saveBox[3]).toBe(Math.round(saveRect.height));
expect(saveBox[0]).toBeGreaterThanOrEqual(Math.round(saveRect.x));
expect(saveBox[1]).toBeGreaterThanOrEqual(Math.round(saveRect.y));
const ref = (snapshot: { elements: { name: string; ref: string }[] }, name: string) =>
snapshot.elements.find((element) => element.name === name)!.ref;
const result = await command("act", {

View File

@ -3,7 +3,6 @@ import {
captureScreenshot,
completeOnboarding,
openNewSpace,
routePatchRpc,
rpc,
signup,
unterhaltungenOnly,
@ -46,30 +45,6 @@ test("Projekte + opens new project dialog instead of group form", async ({ page
await expect(page.getByTestId("project-start-main")).toHaveText(/Starten/);
});
for (const owner of [true, false]) {
const role = owner ? "owner" : "member";
const title = owner
? "new project dialog offers a host folder to the deployment owner"
: "new project dialog offers no host folder to a member";
test(title, async ({ page }, testInfo) => {
await signup(page, `proj-folder-${role}-${Date.now()}@rakazo.test`, "password12", "Folder");
await completeOnboarding(page);
// Whoever signs up first owns the deployment, which depends on spec order.
await routePatchRpc(page, "bootstrap", (json) => ({
...json,
me: { ...(json.me as Record<string, unknown>), isDeploymentOwner: owner },
}));
await page.goto("/app");
await page.waitForURL(/\/app\/[^/]+$/);
await page.locator(".sidebar-projekte").getByTestId("create-project-trigger").click();
const dialog = page.getByRole("dialog", { name: "New project" });
await expect(dialog.getByLabel("Name")).toBeVisible();
await expect(dialog.getByLabel("Project folder")).toHaveCount(owner ? 1 : 0);
await captureScreenshot(page, testInfo, `new-project-${role}`);
});
}
test("spaces stay invisible by default and chat creation requires approval", async ({
page,
}, testInfo) => {

View File

@ -1,317 +0,0 @@
import { useLingui } from "@lingui/react/macro";
import type {
RakazoDesktopBrowserPreview,
RakazoDesktopBrowserPreviewState,
} from "@rakazo/contracts";
import { Button, cn, Input } from "@rakazo/ui-web";
import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react";
import { type FormEvent, useCallback, useEffect, useRef, useState } from "react";
import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url";
import { desktopBridge } from "../../lib/desktop";
const LAST_URL_KEY = "rk:in-app-browser:url";
function readLastUrl(): string {
try {
return window.localStorage.getItem(LAST_URL_KEY) ?? "";
} catch {
return "";
}
}
function rememberUrl(url: string) {
try {
window.localStorage.setItem(LAST_URL_KEY, url);
} catch {
/* A per-viewer convenience only. */
}
}
const EMPTY_STATE: RakazoDesktopBrowserPreviewState = {
url: "",
title: "",
loading: false,
canGoBack: false,
canGoForward: false,
error: null,
};
/** DOM layers the native desktop view would otherwise paint over. */
const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]';
/**
* Address bar plus page. On desktop the page is a native Chromium view laid over the reserved
* area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe.
*/
export function InAppBrowser({ onClose }: { onClose: () => void }) {
const { t } = useLingui();
const native = desktopBridge()?.browserPreview;
const [state, setState] = useState<RakazoDesktopBrowserPreviewState>(() => ({
...EMPTY_STATE,
url: native ? "" : readLastUrl(),
}));
const [draft, setDraft] = useState(state.url);
const [invalid, setInvalid] = useState(false);
const [frameKey, setFrameKey] = useState(0);
const editing = useRef(false);
const inputRef = useRef<HTMLInputElement>(null);
useEffect(() => {
if (!native) return;
let live = true;
const apply = (next: RakazoDesktopBrowserPreviewState) => {
if (!live) return;
setState(next);
if (!editing.current) setDraft(next.url);
};
const unsubscribe = native.onState(apply);
void native
.state()
.then((current) => {
apply(current);
const last = readLastUrl();
if (!current.url && last) void native.navigate(last).catch(() => undefined);
})
.catch(() => undefined);
return () => {
live = false;
unsubscribe();
};
}, [native]);
useEffect(() => {
if (!state.url) inputRef.current?.focus();
}, [state.url]);
const go = useCallback(
(input: string) => {
const url = normalizeBrowserUrl(input);
setInvalid(!url);
if (!url) return;
editing.current = false;
setDraft(url);
rememberUrl(url);
if (native) {
void native.navigate(url).catch(() => setInvalid(true));
} else {
setState((current) => ({ ...current, url, loading: true, error: null }));
setFrameKey((key) => key + 1);
}
},
[native],
);
const reload = useCallback(() => {
if (native) {
void (state.loading ? native.stop() : native.reload()).catch(() => undefined);
return;
}
if (!state.url) return;
setState((current) => ({ ...current, loading: true }));
setFrameKey((key) => key + 1);
}, [native, state.loading, state.url]);
const onSubmit = (event: FormEvent) => {
event.preventDefault();
go(draft);
inputRef.current?.blur();
};
return (
<div className="flex h-full min-h-0 flex-col" data-testid="in-app-browser">
<form onSubmit={onSubmit} className="relative flex items-center gap-0.5 border-b px-2 py-1.5">
{native ? (
<>
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Back`}
disabled={!state.canGoBack}
onClick={() => void native.back().catch(() => undefined)}
>
<ArrowLeft strokeWidth={1.7} className="size-icon-md" />
</Button>
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Forward`}
disabled={!state.canGoForward}
onClick={() => void native.forward().catch(() => undefined)}
>
<ArrowRight strokeWidth={1.7} className="size-icon-md" />
</Button>
</>
) : null}
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={state.loading ? t`Stop` : t`Reload`}
disabled={!state.url}
onClick={reload}
>
{state.loading && native ? (
<X strokeWidth={1.8} className="size-icon-md" />
) : (
<RotateCw strokeWidth={1.7} className="size-icon-md" />
)}
</Button>
<Input
ref={inputRef}
value={draft}
aria-label={t`Address`}
aria-invalid={invalid || undefined}
inputMode="url"
autoComplete="off"
autoCapitalize="off"
spellCheck={false}
placeholder="localhost:5173"
onFocus={(event) => {
editing.current = true;
event.currentTarget.select();
}}
onBlur={() => {
editing.current = false;
}}
onChange={(event) => {
setDraft(event.target.value);
setInvalid(false);
}}
onKeyDown={(event) => {
if (event.key !== "Escape") return;
setDraft(state.url);
setInvalid(false);
event.currentTarget.blur();
}}
className="mx-1 h-control-sm flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm"
/>
{native ? (
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Developer tools`}
disabled={!state.url}
onClick={() => void native.openDevTools().catch(() => undefined)}
>
<Code strokeWidth={1.7} className="size-icon-md" />
</Button>
) : null}
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Open in browser`}
disabled={!state.url}
onClick={() => window.open(state.url, "_blank", "noopener,noreferrer")}
>
<ExternalLink strokeWidth={1.7} className="size-icon-md" />
</Button>
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Close panel`}
onClick={onClose}
>
<X strokeWidth={1.8} className="size-icon-md" />
</Button>
<div
aria-hidden
className={cn(
"absolute inset-x-0 -bottom-px h-0.5 overflow-hidden transition-opacity duration-300",
state.loading ? "opacity-100" : "opacity-0",
)}
>
<div className="rk-browser-progress h-full w-1/3 rounded-full bg-primary/70" />
</div>
</form>
<div className="relative min-h-0 flex-1 bg-background">
{native ? (
<NativeViewport browser={native} visible={Boolean(state.url) && !state.error} />
) : state.url ? (
<iframe
key={frameKey}
title={state.title || t`Page`}
src={state.url}
sandbox={browserFrameSandbox(state.url, window.location.origin)}
allow="clipboard-read; clipboard-write; fullscreen"
onLoad={() => setState((current) => ({ ...current, loading: false }))}
className="h-full w-full border-0 bg-background"
data-testid="in-app-browser-frame"
/>
) : null}
{state.error ? (
<div
role="alert"
className="absolute inset-0 grid place-items-center px-6 text-center text-ui-body-sm text-muted-foreground"
>
{state.error}
</div>
) : null}
</div>
</div>
);
}
/**
* Reserves the page area and keeps the native view glued to it while the panel resizes,
* hiding it whenever app UI (dialogs, menus) floats where the view would paint over it.
*/
function NativeViewport({
browser,
visible,
}: {
browser: RakazoDesktopBrowserPreview;
visible: boolean;
}) {
const ref = useRef<HTMLDivElement>(null);
useEffect(() => {
const element = ref.current;
if (!element) return;
let last = "";
let frame = 0;
const sync = () => {
frame = 0;
const rect = element.getBoundingClientRect();
const covered = Array.from(document.querySelectorAll(FLOATING_UI)).some(
(node) => !element.contains(node),
);
const show = visible && rect.width >= 1 && rect.height >= 1 && !covered;
const bounds = {
x: Math.round(rect.left),
y: Math.round(rect.top),
width: Math.round(rect.width),
height: Math.round(rect.height),
};
const key = show ? JSON.stringify(bounds) : "hidden";
if (key === last) return;
last = key;
void (show ? browser.show(bounds) : browser.hide()).catch(() => undefined);
};
const schedule = () => {
if (!frame) frame = requestAnimationFrame(sync);
};
const resize = new ResizeObserver(schedule);
resize.observe(element);
const floating = new MutationObserver(schedule);
floating.observe(document.body, { childList: true, subtree: true });
window.addEventListener("resize", schedule);
// Sibling layout (sidebars, transitions) can move the area without resizing it.
const settle = setInterval(schedule, 250);
schedule();
return () => {
cancelAnimationFrame(frame);
resize.disconnect();
floating.disconnect();
window.removeEventListener("resize", schedule);
clearInterval(settle);
void browser.hide().catch(() => undefined);
};
}, [browser, visible]);
return <div ref={ref} className="absolute inset-0" data-testid="in-app-browser-viewport" />;
}

View File

@ -1,165 +0,0 @@
import { screenPointInViewer } from "@rakazo/core";
import { Keyboard } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import {
agentPointerFeed,
POINTER_IDLE_MS,
type PointerPoint,
type PointerScreen,
type PointerSegment,
planPointerTimeline,
pointerFrameAt,
} from "../../lib/agent-pointer";
function prefersReducedMotion() {
return (
typeof window.matchMedia === "function" &&
window.matchMedia("(prefers-reduced-motion: reduce)").matches
);
}
/**
* Draws a bot's announced pointer over its live screen: it glides to each target, ripples on
* clicks and shows when it types. Frames are written straight to the DOM so the chat never
* re-renders at animation rate.
*/
export function AgentCursor({
botId,
color,
name,
}: {
botId: string;
color: string;
name: string;
}) {
const layerRef = useRef<HTMLDivElement>(null);
const cursorRef = useRef<HTMLDivElement>(null);
const rippleRef = useRef<HTMLSpanElement>(null);
const [typing, setTyping] = useState(false);
useEffect(() => {
const layer = layerRef.current;
const cursor = cursorRef.current;
const ripple = rippleRef.current;
if (!layer || !cursor || !ripple) return;
const reducedMotion = prefersReducedMotion();
let screen: PointerScreen | null = null;
let segments: PointerSegment[] = [];
let position: PointerPoint | null = null;
let frame = 0;
let idleTimer: ReturnType<typeof setTimeout> | undefined;
let viewer = { width: layer.clientWidth, height: layer.clientHeight };
let shownTyping = false;
const place = (point: PointerPoint, action: string | null, progress: number) => {
if (!screen) return;
const projected = screenPointInViewer(point, viewer, screen);
cursor.style.transform = `translate3d(${projected.x}px, ${projected.y}px, 0)`;
cursor.dataset.action = action ?? "move";
const pressing = action === "click" || action === "double_click" || action === "right_click";
const pulse = action === "double_click" ? (progress * 2) % 1 : progress;
ripple.style.opacity = pressing && !reducedMotion ? String(0.55 * (1 - pulse)) : "0";
ripple.style.transform = `scale(${0.35 + pulse * 1.15})`;
cursor.style.setProperty("--agent-cursor-press", pressing && pulse < 0.35 ? "0.86" : "1");
const nextTyping = action === "type" || action === "key";
if (nextTyping !== shownTyping) {
shownTyping = nextTyping;
setTyping(nextTyping);
}
};
const rest = () => {
clearTimeout(idleTimer);
idleTimer = setTimeout(() => {
cursor.style.opacity = "0";
}, POINTER_IDLE_MS);
};
const tick = () => {
frame = 0;
const now = performance.now();
const state = pointerFrameAt(segments, now, reducedMotion);
if (!state) return;
position = state.point;
place(state.point, state.action?.kind ?? null, state.action?.progress ?? 0);
if (state.done) {
segments = [];
rest();
return;
}
frame = requestAnimationFrame(tick);
};
const unsubscribe = agentPointerFeed.subscribe(botId, (payload) => {
const now = performance.now();
const current = pointerFrameAt(segments, now, reducedMotion)?.point ?? position;
const sameScreen =
screen?.width === payload.screen.width && screen?.height === payload.screen.height;
screen = payload.screen;
// A first announcement starts at its target rather than flying in from a corner.
const firstTarget = payload.steps.find((step) => typeof step.x === "number");
const from =
current && sameScreen
? current
: firstTarget
? { x: firstTarget.x ?? 0, y: firstTarget.y ?? 0 }
: { x: payload.screen.width / 2, y: payload.screen.height / 2 };
segments = planPointerTimeline(from, payload.steps, now);
clearTimeout(idleTimer);
cursor.style.opacity = "1";
if (!frame) frame = requestAnimationFrame(tick);
});
const resize = new ResizeObserver(() => {
viewer = { width: layer.clientWidth, height: layer.clientHeight };
if (position && !frame) place(position, null, 0);
});
resize.observe(layer);
return () => {
unsubscribe();
resize.disconnect();
cancelAnimationFrame(frame);
clearTimeout(idleTimer);
};
}, [botId]);
return (
<div
ref={layerRef}
aria-hidden
data-testid="agent-cursor-layer"
className="pointer-events-none absolute inset-0 overflow-hidden"
>
<div
ref={cursorRef}
data-testid="agent-cursor"
className="absolute top-0 left-0 opacity-0 transition-opacity duration-300 will-change-transform"
style={{ color }}
>
<span
ref={rippleRef}
className="absolute -top-5 -left-5 size-10 rounded-full border-2 border-current opacity-0"
/>
<svg
viewBox="0 0 20 22"
className="relative block size-5 origin-top-left drop-shadow-md transition-transform duration-100"
style={{ transform: "scale(var(--agent-cursor-press, 1))" }}
>
<path
d="M1.5 1.5 L1.5 17.5 L6 13.2 L9.2 20.2 L12.4 18.8 L9.3 12 L15.5 12 Z"
fill="currentColor"
className="stroke-background"
strokeWidth="1.6"
strokeLinejoin="round"
/>
</svg>
<span className="absolute top-5 left-3.5 flex max-w-40 items-center gap-1 rounded-full bg-popover px-2 py-0.5 text-ui-label font-medium whitespace-nowrap text-popover-foreground shadow-sm ring-1 ring-border">
<span className="size-1.5 shrink-0 rounded-full bg-current" style={{ color }} />
<span className="truncate">{name}</span>
{typing ? <Keyboard strokeWidth={1.8} className="size-3 shrink-0" /> : null}
</span>
</div>
</div>
);
}

View File

@ -1,88 +0,0 @@
import { describe, expect, it } from "vitest";
import {
agentPointerFeed,
moveDurationMs,
planPointerTimeline,
pointerFrameAt,
} from "./agent-pointer";
const payload = {
kind: "pointer",
screen: { width: 1280, height: 800 },
steps: [{ action: "click", x: 100, y: 100 }],
};
describe("agentPointerFeed", () => {
it("delivers valid live announcements only to the bot's viewers", () => {
const seen: unknown[] = [];
const other: unknown[] = [];
const stop = agentPointerFeed.subscribe("bot-a", (value) => seen.push(value));
const stopOther = agentPointerFeed.subscribe("bot-b", (value) => other.push(value));
try {
const now = Date.parse("2026-01-01T00:00:10Z");
expect(agentPointerFeed.publish("bot-a", payload, "2026-01-01T00:00:05Z", now)).toBe(true);
// Replayed history and malformed payloads never move the cursor.
expect(agentPointerFeed.publish("bot-a", payload, "2025-12-31T23:00:00Z", now)).toBe(false);
expect(agentPointerFeed.publish("bot-a", { ...payload, steps: [] }, undefined, now)).toBe(
false,
);
expect(agentPointerFeed.publish("bot-a", { kind: "status" }, undefined, now)).toBe(false);
expect(seen).toEqual([payload]);
expect(other).toEqual([]);
} finally {
stop();
stopOther();
}
expect(agentPointerFeed.publish("bot-a", payload)).toBe(true);
expect(seen).toHaveLength(1);
});
});
describe("pointer timeline", () => {
it("travels, acts in place for keyboard steps, then rests", () => {
const segments = planPointerTimeline(
{ x: 0, y: 0 },
[{ action: "click", x: 300, y: 400 }, { action: "type" }, { action: "move", x: 300, y: 400 }],
1_000,
);
expect(segments.map((segment) => segment.to)).toEqual([
{ x: 300, y: 400 },
{ x: 300, y: 400 },
{ x: 300, y: 400 },
]);
const [click, type, move] = segments;
expect(click!.arrive - click!.start).toBe(moveDurationMs(500));
expect(type!.arrive).toBe(type!.start);
expect(move!.end).toBe(move!.start);
const midway = pointerFrameAt(segments, (click!.start + click!.arrive) / 2)!;
expect(midway.action).toBeNull();
// The bow lifts the cursor off the straight line between the points.
const onLine = { x: 150, y: 200 };
expect(Math.hypot(midway.point.x - onLine.x, midway.point.y - onLine.y)).toBeGreaterThan(10);
const clicking = pointerFrameAt(segments, click!.arrive + 1)!;
expect(clicking.point).toEqual({ x: 300, y: 400 });
expect(clicking.action?.kind).toBe("click");
const typing = pointerFrameAt(segments, type!.arrive + 10)!;
expect(typing.action?.kind).toBe("type");
expect(pointerFrameAt(segments, move!.end + 1)).toEqual({
point: { x: 300, y: 400 },
action: null,
done: true,
});
});
it("jumps straight to the target when motion is reduced", () => {
const segments = planPointerTimeline({ x: 0, y: 0 }, [{ action: "move", x: 50, y: 60 }], 0);
expect(pointerFrameAt(segments, 1, true)?.point).toEqual({ x: 50, y: 60 });
});
it("bounds travel time so long and short moves both feel responsive", () => {
expect(moveDurationMs(0)).toBe(0);
expect(moveDurationMs(5)).toBe(180);
expect(moveDurationMs(10_000)).toBe(650);
});
});

View File

@ -1,156 +0,0 @@
import {
type ComputerPointerAction,
ComputerPointerPayload,
type ComputerPointerStep,
} from "@rakazo/contracts";
export type PointerPoint = { x: number; y: number };
export type PointerScreen = ComputerPointerPayload["screen"];
/** Older announcements describe actions that already finished; replaying them would mislead. */
export const POINTER_EVENT_MAX_AGE_MS = 30_000;
export const POINTER_IDLE_MS = 3_500;
type Listener = (payload: ComputerPointerPayload) => void;
/**
* Live pointer announcements per bot. The thread subscription publishes `thread.computer`
* pointer events here; screen viewers subscribe for the bot they show.
*/
class AgentPointerFeed {
private readonly listeners = new Map<string, Set<Listener>>();
publish(botId: string, payload: unknown, createdAt?: string, now = Date.now()): boolean {
const parsed = ComputerPointerPayload.safeParse(payload);
if (!parsed.success) return false;
const at = createdAt ? Date.parse(createdAt) : Number.NaN;
if (Number.isFinite(at) && now - at > POINTER_EVENT_MAX_AGE_MS) return false;
for (const listener of this.listeners.get(botId) ?? []) listener(parsed.data);
return true;
}
subscribe(botId: string, listener: Listener): () => void {
let set = this.listeners.get(botId);
if (!set) {
set = new Set();
this.listeners.set(botId, set);
}
set.add(listener);
return () => {
set.delete(listener);
if (set.size === 0) this.listeners.delete(botId);
};
}
}
export const agentPointerFeed = new AgentPointerFeed();
export type PointerSegment = {
from: PointerPoint;
to: PointerPoint;
/** Perpendicular bow in screen pixels so long moves read as a hand, not a teleport. */
bow: number;
start: number;
arrive: number;
end: number;
action: ComputerPointerAction;
};
const ACTION_DWELL_MS: Record<ComputerPointerAction, number> = {
move: 0,
click: 320,
double_click: 440,
right_click: 320,
down: 160,
up: 160,
type: 700,
key: 280,
scroll: 420,
};
export function moveDurationMs(distance: number): number {
return distance < 1 ? 0 : Math.round(Math.min(650, Math.max(180, 140 + distance * 0.45)));
}
/**
* Lays announced steps on a timeline that starts where the cursor currently is. Keyboard
* steps carry no coordinates and act in place.
*/
export function planPointerTimeline(
from: PointerPoint,
steps: readonly ComputerPointerStep[],
startAt: number,
): PointerSegment[] {
const segments: PointerSegment[] = [];
let position = from;
let time = startAt;
for (const step of steps) {
const to =
typeof step.x === "number" && typeof step.y === "number"
? { x: step.x, y: step.y }
: position;
const distance = Math.hypot(to.x - position.x, to.y - position.y);
const arrive = time + moveDurationMs(distance);
const end = arrive + ACTION_DWELL_MS[step.action];
segments.push({
from: position,
to,
bow: Math.min(48, distance * 0.12),
start: time,
arrive,
end,
action: step.action,
});
position = to;
time = end;
}
return segments;
}
export type PointerFrame = {
point: PointerPoint;
/** The action being performed at the target, with 0..1 progress; null while travelling. */
action: { kind: ComputerPointerAction; progress: number } | null;
done: boolean;
};
function easeInOutCubic(t: number): number {
return t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2;
}
export function pointerFrameAt(
segments: readonly PointerSegment[],
now: number,
reducedMotion = false,
): PointerFrame | null {
if (segments.length === 0) return null;
const last = segments[segments.length - 1]!;
if (now >= last.end) return { point: last.to, action: null, done: true };
const segment = segments.find((candidate) => now < candidate.end) ?? last;
if (now < segment.arrive) {
if (reducedMotion) return { point: segment.to, action: null, done: false };
const span = segment.arrive - segment.start;
const t = easeInOutCubic(span > 0 ? Math.max(0, (now - segment.start) / span) : 1);
const dx = segment.to.x - segment.from.x;
const dy = segment.to.y - segment.from.y;
const length = Math.hypot(dx, dy) || 1;
const lift = Math.sin(Math.PI * t) * segment.bow;
return {
point: {
x: segment.from.x + dx * t - (dy / length) * lift,
y: segment.from.y + dy * t + (dx / length) * lift,
},
action: null,
done: false,
};
}
const dwell = segment.end - segment.arrive;
return {
point: segment.to,
action:
segment.action === "move"
? null
: { kind: segment.action, progress: dwell > 0 ? (now - segment.arrive) / dwell : 1 },
done: false,
};
}

View File

@ -0,0 +1,36 @@
import { describe, expect, it } from "vitest";
import { browserPreviewActivity } from "./browser-preview-context";
describe("browserPreviewActivity", () => {
it("publishes cursors to subscribers and drops cleared ids", () => {
for (const id of browserPreviewActivity.snapshot().keys()) browserPreviewActivity.clear(id);
const seen: string[][] = [];
const unsubscribe = browserPreviewActivity.subscribe((activities) => {
seen.push([...activities.keys()]);
});
try {
browserPreviewActivity.set("bot-a", 50, 40, "Ada");
browserPreviewActivity.set("bot-a", 50, 40, "Ada");
browserPreviewActivity.set("bot-b", 10, 20, "Bob");
expect(browserPreviewActivity.snapshot().get("bot-a")).toEqual({
x: 50,
y: 40,
label: "Ada",
});
browserPreviewActivity.clear("bot-a");
browserPreviewActivity.clear("missing");
expect(seen).toEqual([[], ["bot-a"], ["bot-a", "bot-b"], ["bot-b"]]);
} finally {
unsubscribe();
for (const id of [...browserPreviewActivity.snapshot().keys()]) {
browserPreviewActivity.clear(id);
}
}
expect(browserPreviewActivity.snapshot().size).toBe(0);
});
});

View File

@ -0,0 +1,47 @@
export type BrowserPreviewCursor = {
x: number;
y: number;
label: string;
};
type Listener = (activities: ReadonlyMap<string, BrowserPreviewCursor>) => void;
/**
* In-memory cursors for the desktop browser-preview panel.
* The shell publishes working bots here; the panel subscribes.
*/
class BrowserPreviewActivity {
private readonly activities = new Map<string, BrowserPreviewCursor>();
private readonly listeners = new Set<Listener>();
subscribe(listener: Listener): () => void {
this.listeners.add(listener);
listener(this.snapshot());
return () => {
this.listeners.delete(listener);
};
}
snapshot(): ReadonlyMap<string, BrowserPreviewCursor> {
return new Map(this.activities);
}
set(id: string, x: number, y: number, label: string) {
const current = this.activities.get(id);
if (current && current.x === x && current.y === y && current.label === label) return;
this.activities.set(id, { x, y, label });
this.emit();
}
clear(id: string) {
if (!this.activities.delete(id)) return;
this.emit();
}
private emit() {
const snapshot = this.snapshot();
for (const listener of this.listeners) listener(snapshot);
}
}
export const browserPreviewActivity = new BrowserPreviewActivity();

View File

@ -1,38 +0,0 @@
import { describe, expect, it } from "vitest";
import { browserFrameSandbox, normalizeBrowserUrl } from "./browser-url";
describe("normalizeBrowserUrl", () => {
it("defaults local dev servers to http and the web to https", () => {
expect(normalizeBrowserUrl("localhost:5173")).toBe("http://localhost:5173/");
expect(normalizeBrowserUrl("127.0.0.1:3000/app?x=1")).toBe("http://127.0.0.1:3000/app?x=1");
expect(normalizeBrowserUrl("example.com/docs")).toBe("https://example.com/docs");
expect(normalizeBrowserUrl(" https://example.com ")).toBe("https://example.com/");
expect(normalizeBrowserUrl("http://example.com")).toBe("http://example.com/");
});
it("rejects input that is not an http(s) address", () => {
for (const input of [
"",
"hello world",
"hello",
"javascript:alert(1)",
"file:///etc/passwd",
"data:text/html,hi",
"ftp://example.com",
"https://user:fake-password@example.com",
]) {
expect(normalizeBrowserUrl(input), input).toBeNull();
}
});
});
describe("browserFrameSandbox", () => {
it("never grants same-origin access to the app's own origin", () => {
expect(browserFrameSandbox("https://app.test/x", "https://app.test")).not.toContain(
"allow-same-origin",
);
expect(browserFrameSandbox("http://localhost:5173/", "https://app.test")).toContain(
"allow-same-origin",
);
});
});

View File

@ -1,39 +0,0 @@
const LOCAL_HOST = /^(localhost|127(?:\.\d{1,3}){3}|\[::1\]|0\.0\.0\.0|[\w-]+\.local(?:host)?)$/i;
/**
* Turns what someone types in the address bar into an http(s) URL. Local dev servers default to
* http, everything else to https. Returns null for input that is not an address.
*/
export function normalizeBrowserUrl(input: string): string | null {
const value = input.trim();
if (!value || /\s/.test(value)) return null;
const hasScheme = /^[a-z][a-z\d+.-]*:\/\//i.test(value);
if (!hasScheme && /^[a-z][a-z\d+.-]*:(?!\d)/i.test(value)) return null;
let candidate = value;
if (!hasScheme) {
const host = value.split(/[/?#]/, 1)[0]!.replace(/:\d+$/, "");
if (!LOCAL_HOST.test(host) && !host.includes(".") && !/^\d+$/.test(host)) return null;
candidate = `${LOCAL_HOST.test(host) ? "http" : "https"}://${value}`;
}
try {
const url = new URL(candidate);
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
if (url.username || url.password || !url.hostname) return null;
return url.toString();
} catch {
return null;
}
}
/**
* Sandbox for the web fallback. A page on the app's own origin must not get same-origin
* access, or it could reach into the app around the sandbox.
*/
export function browserFrameSandbox(url: string, appOrigin: string): string {
const base = "allow-scripts allow-forms allow-popups allow-modals allow-downloads";
try {
return new URL(url).origin === appOrigin ? base : `${base} allow-same-origin`;
} catch {
return base;
}
}

View File

@ -925,24 +925,6 @@ describe("thread event reduction", () => {
expect(next?.messages).toEqual(initial.messages);
});
it("advances past a bot pointer announcement without touching messages", () => {
const initial = snapshot([message("m1", [{ kind: "text", text: "hi" }], 3)], 3);
const pointer = event({
type: "thread.computer",
seq: 4,
runId: "run-1",
payload: {
kind: "pointer",
screen: { width: 1280, height: 800 },
steps: [{ action: "click", x: 10, y: 20 }],
},
});
expect(isThreadSnapshotEvent(pointer)).toBe(true);
const next = reduceThreadSnapshot(initial, pointer);
expect(next?.cursor).toBe(4);
expect(next?.messages).toEqual(initial.messages);
});
it("holds a tool call that lands mid-sentence until the sentence completes", () => {
const initial = snapshot([]);

View File

@ -249,7 +249,6 @@ export function isThreadSnapshotEvent(event: ProductEvent): boolean {
event.type === "thread.cloud_agent" ||
event.type === "agent.tool.called" ||
event.type === "agent.tool.completed" ||
event.type === "thread.computer" ||
event.type === "thread.message.created" ||
event.type === "thread.message.updated" ||
event.type === "thread.message.reaction" ||
@ -440,7 +439,7 @@ export function reduceThreadSnapshot(
};
return { ...prev, cursor: event.seq, messages: [...remaining, next] };
}
if (event.type === "agent.tool.completed" || event.type === "thread.computer") {
if (event.type === "agent.tool.completed") {
return { ...prev, cursor: event.seq };
}
if (event.type === "thread.subagent") {

View File

@ -110,7 +110,6 @@ import {
Copy,
Folder,
Gauge,
Globe,
GripVertical,
LayoutGrid,
Lock,
@ -160,7 +159,6 @@ import { AskCard } from "../components/AskCard";
import { ActiveBotGlyph, CollaborationMarker } from "../components/ai/CollaborationMarker";
import { DelegatedTaskCard, SentMessageCard } from "../components/ai/DelegatedTaskCard";
import { ProjectCreatedCard } from "../components/ai/ProjectCreatedCard";
import { InAppBrowser } from "../components/browser/InAppBrowser";
import { CloudAgentCard } from "../components/CloudAgentCard";
import { ComputerMaintenanceActions } from "../components/ComputerMaintenanceActions";
import {
@ -168,7 +166,6 @@ import {
computersAreUnavailable,
} from "../components/ComputersUnavailableHint";
import { ComputerUpdateProgress } from "../components/ComputerUpdateProgress";
import { AgentCursor } from "../components/computer/AgentCursor";
import { DataTrustPanel } from "../components/DataTrustPanel";
import { MainInbox } from "../components/MainInbox";
import { MessageHoverMetadata } from "../components/MessageHoverMetadata";
@ -182,7 +179,6 @@ import { VoiceAuroraRim } from "../components/voice/VoiceAurora";
import { useSidebarResize } from "../hooks/useSidebarResize";
import { readActivityMode, writeActivityMode } from "../lib/activity-mode";
import { needsYouChipLabel, rosterActivitySubtitle, rosterStatusDot } from "../lib/agent-activity";
import { agentPointerFeed } from "../lib/agent-pointer";
import type { ArtifactTarget } from "../lib/artifact-open";
import { authClient } from "../lib/auth";
import { takeInitialBootstrap } from "../lib/bootstrap";
@ -196,6 +192,8 @@ import {
requestBrowserNotificationPermission,
shouldNotifyBrowser,
} from "../lib/browser-notifications";
import type { BrowserPreviewCursor } from "../lib/browser-preview-context";
import { browserPreviewActivity } from "../lib/browser-preview-context";
import { loadComputerScreen } from "../lib/computer-screen";
import { desktopBridge } from "../lib/desktop";
import { scheduleFocusPrompt } from "../lib/focus-prompt";
@ -383,6 +381,31 @@ function readCollapsedSidebarSections(userId: string | null | undefined): Set<st
}
}
function copyBrowserPreviewActivities(
activities: ReadonlyMap<string, BrowserPreviewCursor>,
): Map<string, BrowserPreviewCursor> {
return new Map(activities);
}
function sameBrowserPreviewActivities(
current: ReadonlyMap<string, BrowserPreviewCursor>,
next: ReadonlyMap<string, BrowserPreviewCursor>,
): boolean {
if (current.size !== next.size) return false;
for (const [id, cursor] of next) {
const existing = current.get(id);
if (
!existing ||
existing.x !== cursor.x ||
existing.y !== cursor.y ||
existing.label !== cursor.label
) {
return false;
}
}
return true;
}
export type NavPillar = "heute" | "team" | "projekte";
function stripSpaceTitlePrefix(title: string | null, spaceName: string): string | null {
@ -457,7 +480,16 @@ export function ShellPage() {
setBotsSidebarCollapsed(readBotsSidebarCollapsed(userId));
}, [userId]);
// ── Browser Preview ──────────────────────────────────────────────
const bridge = desktopBridge();
const [isDesktop] = useState(Boolean(bridge?.browserPreview));
const [panel, setPanel] = useState<Panel>(null);
const [browserUrl, setBrowserUrl] = useState("http://localhost:5173");
const [devServerRunning, setDevServerRunning] = useState(false);
const [browserLoading, setBrowserLoading] = useState(false);
const [agentActivities, setAgentActivities] = useState<
Map<string, { x: number; y: number; label: string }>
>(new Map());
const [showCreateBotModal, setShowCreateBotModal] = useState(false);
@ -473,6 +505,60 @@ export function ShellPage() {
useSensor(TouchSensor, { activationConstraint: { distance: 8 } }),
);
// Browser Preview lifecycle
useEffect(() => {
const bp = bridge?.browserPreview;
if (!bp) return;
const unsubNav = bp.onNavigate((url) => setBrowserUrl(url));
const unsubLoading = bp.onLoadingProgress((p) => setBrowserLoading(p < 1));
bp.getState()
.then((s) => {
setBrowserUrl(s.url);
setDevServerRunning(s.devServerRunning);
})
.catch(() => {});
return () => {
unsubNav();
unsubLoading();
};
}, [bridge]);
const startDevServer = useCallback(async () => {
try {
await bridge?.browserPreview?.devServer.start();
setDevServerRunning(true);
} catch {
/* */
}
}, [bridge]);
const stopDevServer = useCallback(async () => {
try {
await bridge?.browserPreview?.devServer.stop();
setDevServerRunning(false);
} catch {
/* */
}
}, [bridge]);
const navigateBrowser = useCallback(
(url: string) => {
bridge?.browserPreview?.navigate(url).catch(() => {});
},
[bridge],
);
const reloadBrowser = useCallback(() => {
bridge?.browserPreview?.reload().catch(() => {});
}, [bridge]);
// Agent activity → browser preview cursors
useEffect(() => {
const unsub = browserPreviewActivity.subscribe((acts) => {
setAgentActivities((current) =>
sameBrowserPreviewActivities(current, acts) ? current : copyBrowserPreviewActivities(acts),
);
});
return unsub;
}, []);
const [query, setQuery] = useState("");
const [searchHits, setSearchHits] = useState<SearchHit[]>([]);
const [searchLoading, setSearchLoading] = useState(false);
@ -1456,8 +1542,6 @@ export function ShellPage() {
void refreshThread(active.id).catch(() => undefined);
} else if (isComputerStatusEvent(event)) {
void refreshComputerScreen(active.id).catch(() => undefined);
} else if (event.type === "thread.computer") {
agentPointerFeed.publish(event.botId, event.payload, event.createdAt);
}
},
});
@ -2153,6 +2237,20 @@ export function ShellPage() {
}));
}, [activeSnapshot, enrichStageMember, inGroup, resolveTranscriptBot, workingRuns]);
// Working bots → browser preview activity cursors
useEffect(() => {
if (workingBots.length === 0) return;
for (const bot of workingBots) {
if (!bot.botId) continue;
const label = bot.name ?? bot.botId.slice(0, 8);
browserPreviewActivity.set(bot.botId, 50, 40, label);
}
const activeBotIds = new Set(workingBots.map((b) => b.botId).filter(Boolean));
for (const id of browserPreviewActivity.snapshot().keys()) {
if (!activeBotIds.has(id)) browserPreviewActivity.clear(id);
}
}, [workingBots]);
const encounterMode = Boolean((active || activeGroup) && !showProjectTimeline && !callOpen);
const voiceEncounterActive = encounterMode || callOpen || voiceEntry;
const voiceCallBot = useMemo(() => {
@ -4394,16 +4492,6 @@ export function ShellPage() {
</button>
</div>
<div className="flex items-center gap-2">
<button
type="button"
title={t`Browser`}
aria-label={t`Browser`}
onClick={() => setPanel(panel === "browser" ? null : "browser")}
data-active={panel === "browser" ? "" : undefined}
className="app-no-drag grid h-[30px] w-[34px] place-items-center rounded-[9px] hover:bg-accent data-active:bg-accent"
>
<Globe strokeWidth={1.7} className="text-foreground/75 size-icon-lg" />
</button>
{!inGroup && active ? (
<button
type="button"
@ -4633,9 +4721,7 @@ export function ShellPage() {
onPointerDown={startResizingRight}
className="absolute -left-2 top-0 bottom-0 w-4 cursor-col-resize z-50 hidden md:block"
/>
{panel === "browser" ? (
<InAppBrowser onClose={() => setPanel(null)} />
) : panel && (active || activeGroup || panel === "create" || panel === "create-group") ? (
{panel && (active || activeGroup || panel === "create" || panel === "create-group") ? (
<div className="rk-scroll h-full w-full overflow-y-auto px-5 lg:px-7 2xl:px-9 py-[17px]">
{panel !== "routine" &&
panel !== "create" &&
@ -4675,6 +4761,17 @@ export function ShellPage() {
<Settings strokeWidth={1.7} className="size-icon-md" />
</Button>
) : null}
{!panel && active ? (
<Button
variant="ghost"
size="icon-sm"
aria-label={t`Browser preview`}
onClick={() => setPanel("browser")}
className="text-muted-foreground"
>
<PanelRight strokeWidth={1.7} className="size-icon-md" />
</Button>
) : null}
<Button
variant="ghost"
size="icon-sm"
@ -4686,6 +4783,80 @@ export function ShellPage() {
</div>
</div>
) : null}
{panel === "browser" ? (
<div className="flex h-full flex-col">
<div className="flex items-center gap-1.5 border-b px-3 py-2">
<Button
variant="ghost"
size="icon-sm"
onClick={reloadBrowser}
aria-label={t`Reload`}
className="size-icon-sm text-muted-foreground"
>
<RefreshCw strokeWidth={1.7} className="size-4" />
</Button>
<input
type="url"
value={browserUrl}
onChange={(e) => setBrowserUrl(e.target.value)}
onBlur={() => navigateBrowser(browserUrl)}
onKeyDown={(e) => {
if (e.key === "Enter") navigateBrowser(browserUrl);
}}
placeholder="http://localhost:5173"
className="flex-1 rounded-md bg-muted/60 px-2.5 py-1 text-xs text-foreground placeholder:text-muted-foreground/60 outline-none focus:ring-1 focus:ring-ring/50"
/>
{isDesktop ? (
devServerRunning ? (
<span className="inline-flex items-center gap-1 rounded-full bg-muted px-2 py-0.5 text-ui-label font-medium text-foreground">
<span className="size-1.5 rounded-full bg-success" />
<Trans>Dev</Trans>
</span>
) : (
<Button
variant="ghost"
size="xs"
onClick={startDevServer}
className="text-ui-label text-muted-foreground hover:text-foreground"
>
<Trans>Start Dev</Trans>
</Button>
)
) : null}
{browserLoading && (
<div className="h-0.5 w-full rounded-full bg-muted/30">
<div className="h-full w-2/3 animate-pulse rounded-full bg-primary/60" />
</div>
)}
</div>
<div className="relative flex-1 overflow-hidden bg-background">
<div className="relative h-full w-full" data-testid="browser-preview">
{agentActivities.size > 0 && (
<div className="pointer-events-none absolute inset-0">
{Array.from(agentActivities.entries()).map(([agentId, activity]) => {
const cx = Math.min(100, Math.max(0, activity.x)) + "%";
const cy = Math.min(100, Math.max(0, activity.y)) + "%";
return (
<div
key={agentId}
className="absolute transition-[left,top] duration-300 ease-out"
style={{ left: cx, top: cy }}
>
<div className="relative">
<div className="size-3 rounded-full bg-primary/60 ring-2 ring-primary" />
<div className="absolute -right-1 -top-4 whitespace-nowrap rounded bg-overlay px-1.5 py-0.5 text-ui-label text-foreground shadow-sm">
{activity.label}
</div>
</div>
</div>
);
})}
</div>
)}
</div>
</div>
</div>
) : null}
{panel === "computer" && active ? (
<div>
<div
@ -4699,17 +4870,14 @@ export function ShellPage() {
) : computer?.kind === "desktop" ? (
<DesktopKindEmptyState className="grid h-full place-items-center px-6 text-center text-sm text-muted-foreground/80" />
) : computer?.state === "running" && embeddedScreenUrl && !computerScreenError ? (
<>
<iframe
title={t`Bot screen preview`}
src={embeddedScreenUrl}
sandbox={screenIframeSandbox(embeddedScreenUrl)}
className="h-full w-full border-0 bg-black"
allow="clipboard-read; clipboard-write"
style={{ pointerEvents: "none" }}
/>
<AgentCursor botId={active.id} color={active.color} name={active.name} />
</>
<iframe
title={t`Bot screen preview`}
src={embeddedScreenUrl}
sandbox={screenIframeSandbox(embeddedScreenUrl)}
className="h-full w-full border-0 bg-black"
allow="clipboard-read; clipboard-write"
style={{ pointerEvents: "none" }}
/>
) : (
<div className="grid h-full place-items-center px-6 text-center text-sm text-muted-foreground/80">
{computerScreenError ??
@ -5314,7 +5482,6 @@ export function ShellPage() {
{newSpaceOpen ? (
<NewSpaceDialog
variant={newSpaceDialogVariant}
canLinkFolder={bootstrapMe?.isDeploymentOwner === true}
onCancel={() => setNewSpaceOpen(false)}
onConfirm={async ({ name, workspacePath }) => {
const space = await rpc.spaces.create({
@ -5600,9 +5767,6 @@ export function ShellPage() {
pointerEvents: recordingSkill || !hasControl ? "none" : "auto",
}}
/>
{active && !hasControl ? (
<AgentCursor botId={active.id} color={active.color} name={active.name} />
) : null}
{active ? (
<TeachCaptureOverlay
botId={active.id}

View File

@ -38,13 +38,10 @@ export type NewSpaceConfirmInput = {
export function NewSpaceDialog({
variant = "space",
canLinkFolder,
onCancel,
onConfirm,
}: {
variant?: "space" | "project";
/** Folders live on the host, which only the deployment owner may link. */
canLinkFolder: boolean;
onCancel: () => void;
onConfirm: (input: NewSpaceConfirmInput) => Promise<void>;
}) {
@ -56,7 +53,6 @@ export function NewSpaceDialog({
const [saving, setSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const isProject = variant === "project";
const showFolder = isProject && canLinkFolder;
const desktop = desktopBridge();
const trimmedWorkspacePath = workspacePath.trim();
@ -67,7 +63,7 @@ export function NewSpaceDialog({
setError(null);
void onConfirm({
name: trimmed,
...(showFolder && trimmedWorkspacePath ? { workspacePath: trimmedWorkspacePath } : {}),
...(isProject && trimmedWorkspacePath ? { workspacePath: trimmedWorkspacePath } : {}),
}).catch((reason: unknown) => {
setError(
reason instanceof Error
@ -108,7 +104,7 @@ export function NewSpaceDialog({
className="mt-2"
/>
</label>
{showFolder ? (
{isProject ? (
<div className="space-y-2">
<label htmlFor={folderId} className="block text-ui-body text-foreground/75">
<Trans>Project folder</Trans>

View File

@ -240,26 +240,6 @@ body,
-webkit-app-region: no-drag;
}
/* In-app browser load indicator: a short bar sweeping under the address bar. */
.rk-browser-progress {
animation: rkBrowserProgress 1.1s cubic-bezier(0.65, 0, 0.35, 1) infinite;
}
@keyframes rkBrowserProgress {
from {
transform: translateX(-100%);
}
to {
transform: translateX(300%);
}
}
@media (prefers-reduced-motion: reduce) {
.rk-browser-progress {
animation: none;
width: 100%;
opacity: 0.6;
}
}
@keyframes rkPulse {
0%,
100% {

View File

@ -412,39 +412,10 @@ EVAL_HELPERS = r"""
})
.join("\n");
}
// Screen pixels of the element center's box, only while that center is in the viewport.
// Drives the visible bot cursor; the model never sees it.
function screenBox(el, viewport) {
const rect = el.getBoundingClientRect();
const cx = rect.left + rect.width / 2;
const cy = rect.top + rect.height / 2;
if (cx < 0 || cy < 0 || cx > window.innerWidth || cy > window.innerHeight) return undefined;
const box = [
viewport.x + rect.left * viewport.scale,
viewport.y + rect.top * viewport.scale,
rect.width * viewport.scale,
rect.height * viewport.scale,
].map(Math.round);
return box[0] >= 0 && box[1] >= 0 && box[2] > 0 && box[3] > 0 ? box : undefined;
}
function viewportOnScreen() {
const scale = window.devicePixelRatio || 1;
// Browser chrome sits above the page; side borders are split evenly.
return {
scale,
x: (window.screenX + Math.max(0, (window.outerWidth - window.innerWidth) / 2)) * scale,
y: (window.screenY + Math.max(0, window.outerHeight - window.innerHeight)) * scale,
screen: {
width: Math.round(window.screen.width * scale),
height: Math.round(window.screen.height * scale),
},
};
}
window.__rakazoPageBrowser = {
snapshot() {
refMap.clear();
const elements = [];
const viewport = viewportOnScreen();
for (const el of document.querySelectorAll("body *")) {
if (!isInteractive(el) || !visible(el)) continue;
const ref = newRef();
@ -457,14 +428,12 @@ EVAL_HELPERS = r"""
: ["checkbox", "radio"].includes(el.type) ? (el.checked ? "checked" : "unchecked")
: "value" in el ? String(el.value ?? "").slice(0, 2000) : undefined,
tag: el.tagName.toLowerCase(),
box: screenBox(el, viewport),
});
if (elements.length >= 80) break;
}
return {
url: location.href,
title: (document.title || "").slice(0, 200),
screen: viewport.screen,
elements,
tree: (document.body?.innerText || "").slice(0, 12000) + "\n" + formatTree(elements)
+ (document.querySelector("iframe,frame") ? "\nEmbedded frames require desktop tools." : ""),

View File

@ -746,21 +746,14 @@ export interface BrowserSnapshotNode {
name: string;
value?: string;
tag?: string;
/** Screen pixels when the element's center is in view; drives the visible bot cursor only. */
box?: BrowserScreenBox;
}
/** `[x, y, width, height]` in screen pixels of the computer display. */
export type BrowserScreenBox = [number, number, number, number];
export interface BrowserSnapshotResult {
url: string;
title: string;
/** Compact accessibility-style tree for the model. */
tree: string;
elements: BrowserSnapshotNode[];
/** Display size the element boxes are relative to. */
screen?: { width: number; height: number };
fallback?: "computer_act";
error?: string;
}
@ -786,7 +779,6 @@ export interface BrowserActResult {
/** Snapshot after the actions when available. */
tree?: string;
elements?: BrowserSnapshotNode[];
screen?: { width: number; height: number };
fallback?: "computer_act";
error?: string;
}

View File

@ -5,14 +5,6 @@ import type {
BrowserProvider,
ComputerRef,
} from "@rakazo/adapter-kit";
import type { ComputerPointerPayload } from "@rakazo/contracts";
import { type BrowserPointerTargets, withoutBrowserGeometry } from "./computer-pointer.js";
/** Optional hooks that let the visible bot cursor follow page-browser actions. */
export interface BrowserPointerHooks {
targets: BrowserPointerTargets;
announce: (payload: ComputerPointerPayload) => Promise<void>;
}
const MAX_BROWSER_ACTIONS = 24;
@ -21,7 +13,6 @@ export async function browserNavigateFromTool(
computer: ComputerRef,
context: AdapterContext,
args: Record<string, unknown>,
pointer?: BrowserPointerHooks,
) {
const url = String(args.url ?? "").trim();
try {
@ -32,7 +23,6 @@ export async function browserNavigateFromTool(
fallback: "computer_act" as const,
};
}
pointer?.targets.forget();
const result = await browser.navigate(computer, { url, signal: context.signal }, context);
return formatBrowserResult(result);
} catch (error) {
@ -49,13 +39,11 @@ export async function browserSnapshotFromTool(
computer: ComputerRef,
context: AdapterContext,
args: Record<string, unknown>,
pointer?: BrowserPointerHooks,
) {
void args;
try {
const result = await browser.snapshot(computer, { signal: context.signal }, context);
pointer?.targets.remember(result);
return formatBrowserResult(withoutBrowserGeometry(result));
return formatBrowserResult(result);
} catch (error) {
context.signal.throwIfAborted();
return {
@ -70,16 +58,11 @@ export async function browserActFromTool(
computer: ComputerRef,
context: AdapterContext,
args: Record<string, unknown>,
pointer?: BrowserPointerHooks,
) {
try {
const actions = parseBrowserActions(args.actions);
const announcement = pointer?.targets.payloadFor(actions);
if (announcement) await pointer?.announce(announcement);
const result = await browser.act(computer, { actions, signal: context.signal }, context);
if (result.elements) pointer?.targets.remember(result);
else pointer?.targets.forget();
return formatBrowserResult(withoutBrowserGeometry(result));
return formatBrowserResult(result);
} catch (error) {
context.signal.throwIfAborted();
return {

View File

@ -115,7 +115,6 @@ export class ComputerBrowserProvider implements BrowserProvider {
title: typeof live.title === "string" ? live.title : "",
tree: typeof live.tree === "string" ? live.tree : formatTree(elements),
elements,
...(live.screen ? { screen: live.screen } : {}),
};
}
@ -151,7 +150,6 @@ export class ComputerBrowserProvider implements BrowserProvider {
title: typeof live.title === "string" ? live.title : "",
tree: typeof live.tree === "string" ? live.tree : elements ? formatTree(elements) : undefined,
elements,
...(live.screen ? { screen: live.screen } : {}),
};
}

View File

@ -1,164 +0,0 @@
import type {
AdapterContext,
BrowserActRequest,
BrowserActResult,
BrowserProvider,
BrowserSnapshotResult,
ComputerRef,
} from "@rakazo/adapter-kit";
import { ComputerPointerPayload } from "@rakazo/contracts";
import { DEFAULT_COMPUTER_SCREEN } from "@rakazo/core";
import { describe, expect, it } from "vitest";
import { browserActFromTool, browserSnapshotFromTool } from "./browser-tools.js";
import {
BrowserPointerTargets,
computerPointerPayload,
computerPointerSteps,
} from "./computer-pointer.js";
import { parseComputerActions } from "./computer-tools.js";
const context: AdapterContext = {
operationId: "1",
traceId: "1",
spaceId: "w",
userId: "u",
signal: new AbortController().signal,
};
const computer: ComputerRef = { id: "c1", botId: "b1", kind: "docker", providerRef: "c1" };
describe("computer pointer steps", () => {
it("maps a computer_act batch to cursor steps in order", () => {
const steps = computerPointerSteps(
parseComputerActions([
{ kind: "move", x: 10, y: 20 },
{ kind: "click", x: 30, y: 40, double: true },
{ kind: "click", x: 50, y: 60, button: "right" },
{ kind: "type", text: "hello" },
{ kind: "key", key: "Enter" },
{ kind: "wait", ms: 100 },
{ kind: "scroll", direction: "down" },
{ kind: "down", x: 1, y: 2 },
{ kind: "up", x: 3, y: 4 },
]),
);
expect(steps).toEqual([
{ action: "move", x: 10, y: 20 },
{ action: "double_click", x: 30, y: 40 },
{ action: "right_click", x: 50, y: 60 },
{ action: "type" },
{ action: "key" },
{ action: "scroll" },
{ action: "down", x: 1, y: 2 },
{ action: "up", x: 3, y: 4 },
]);
const payload = computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps);
expect(ComputerPointerPayload.parse(payload)).toEqual(payload);
});
it("keeps two separate clicks at different points apart", () => {
expect(
computerPointerSteps(
parseComputerActions([
{ kind: "click", x: 1, y: 1 },
{ kind: "click", x: 2, y: 2 },
]),
),
).toEqual([
{ action: "click", x: 1, y: 1 },
{ action: "click", x: 2, y: 2 },
]);
});
it("announces nothing for a batch without visible steps", () => {
const steps = computerPointerSteps(parseComputerActions([{ kind: "wait", ms: 10 }]));
expect(computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps)).toBeUndefined();
});
});
describe("browser pointer targets", () => {
it("points at element centers and stops at the first unknown ref", () => {
const targets = new BrowserPointerTargets();
targets.remember({
screen: { width: 1280, height: 800 },
elements: [
{ ref: "e1", role: "textbox", name: "Name", box: [100, 200, 40, 20] },
{ ref: "e2", role: "button", name: "Save", box: [300, 400, 11, 11] },
{ ref: "e3", role: "button", name: "Hidden" },
// Untrusted helper output never becomes a cursor target.
{ ref: "e4", role: "button", name: "Bad", box: [-5, 0, 10, 10] as never },
],
});
expect(
targets.payloadFor([
{ kind: "fill", ref: "e1", text: "x" },
{ kind: "click", ref: "e2" },
{ kind: "click", ref: "e3" },
{ kind: "click", ref: "e1" },
]),
).toEqual({
kind: "pointer",
screen: { width: 1280, height: 800 },
steps: [
{ action: "type", x: 120, y: 210 },
{ action: "click", x: 306, y: 406 },
],
});
expect(targets.payloadFor([{ kind: "click", ref: "e4" }])).toBeUndefined();
targets.forget();
expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
});
it("needs a reported screen size", () => {
const targets = new BrowserPointerTargets();
targets.remember({ elements: [{ ref: "e1", role: "button", name: "A", box: [1, 1, 2, 2] }] });
expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
});
});
describe("page browser tools with a visible cursor", () => {
it("announces before acting and keeps geometry away from the model", async () => {
const order: string[] = [];
const snapshot: BrowserSnapshotResult = {
url: "https://example.test/",
title: "Example",
tree: '- button "Save" [e1]',
screen: { width: 1280, height: 800 },
elements: [{ ref: "e1", role: "button", name: "Save", box: [10, 20, 30, 40] }],
};
const browser = {
snapshot: async () => snapshot,
act: async (_computer: ComputerRef, request: BrowserActRequest) => {
order.push(`act:${request.actions.length}`);
return { ok: true, completed: 1, ...snapshot } satisfies BrowserActResult;
},
} as unknown as BrowserProvider;
const announced: unknown[] = [];
const pointer = {
targets: new BrowserPointerTargets(),
announce: async (payload: ComputerPointerPayload) => {
order.push("announce");
announced.push(payload);
},
};
const seen = await browserSnapshotFromTool(browser, computer, context, {}, pointer);
expect(JSON.stringify(seen)).not.toMatch(/box|screen/);
const acted = await browserActFromTool(
browser,
computer,
context,
{ actions: [{ kind: "click", ref: "e1" }] },
pointer,
);
expect(order).toEqual(["announce", "act:1"]);
expect(announced).toEqual([
{
kind: "pointer",
screen: { width: 1280, height: 800 },
steps: [{ action: "click", x: 25, y: 40 }],
},
]);
expect(JSON.stringify(acted)).not.toMatch(/box|screen/);
});
});

View File

@ -1,130 +0,0 @@
import type {
BrowserActStep,
BrowserScreenBox,
BrowserSnapshotNode,
ComputerAction,
} from "@rakazo/adapter-kit";
import {
type ComputerPointerPayload,
type ComputerPointerStep,
MAX_COMPUTER_POINTER_STEPS,
} from "@rakazo/contracts";
export type ComputerScreenSize = ComputerPointerPayload["screen"];
/** Pointer steps for a validated computer_act batch, in execution order. */
export function computerPointerSteps(actions: ComputerAction[]): ComputerPointerStep[] {
const steps: ComputerPointerStep[] = [];
for (let index = 0; index < actions.length; index += 1) {
const action = actions[index]!;
if (action.kind === "pointer") {
const next = actions[index + 1];
// parseComputerActions expands a double click into two identical clicks.
if (
action.type === "click" &&
next?.kind === "pointer" &&
next.type === "click" &&
next.x === action.x &&
next.y === action.y &&
next.button === action.button
) {
steps.push({ action: "double_click", x: action.x, y: action.y });
index += 1;
continue;
}
const kind =
action.type === "click"
? action.button === "right"
? "right_click"
: "click"
: action.type;
steps.push({ action: kind, x: action.x, y: action.y });
} else if (action.kind === "clipboard") {
steps.push({ action: "type" });
} else if (action.kind === "key" || action.kind === "scroll") {
steps.push({ action: action.kind });
}
}
return steps.slice(0, MAX_COMPUTER_POINTER_STEPS);
}
export function computerPointerPayload(
screen: ComputerScreenSize,
steps: ComputerPointerStep[],
): ComputerPointerPayload | undefined {
if (steps.length === 0) return undefined;
return { kind: "pointer", screen, steps: steps.slice(0, MAX_COMPUTER_POINTER_STEPS) };
}
/**
* Remembers where the page browser last saw each element on screen, so a browser_act batch
* can be announced as pointer steps before it runs. Geometry never reaches the model.
*/
export class BrowserPointerTargets {
private boxes = new Map<string, BrowserScreenBox>();
private screen: ComputerScreenSize | undefined;
remember(result: { elements?: BrowserSnapshotNode[]; screen?: ComputerScreenSize }) {
if (!Array.isArray(result.elements)) return;
this.boxes = new Map();
this.screen = validScreen(result.screen);
for (const element of result.elements) {
if (validBox(element.box)) this.boxes.set(element.ref, element.box);
}
}
forget() {
this.boxes = new Map();
this.screen = undefined;
}
payloadFor(actions: BrowserActStep[]): ComputerPointerPayload | undefined {
if (!this.screen) return undefined;
const steps: ComputerPointerStep[] = [];
for (const action of actions) {
const box = this.boxes.get(action.ref);
// Later steps may target elements an earlier step reveals; stop at the first unknown one.
if (!box) break;
const [x, y, width, height] = box;
steps.push({
action: action.kind === "click" ? "click" : "type",
x: Math.round(x + width / 2),
y: Math.round(y + height / 2),
});
}
return computerPointerPayload(this.screen, steps);
}
}
/** Strips screen geometry from page-browser results before they reach the model. */
export function withoutBrowserGeometry<T extends { elements?: BrowserSnapshotNode[] }>(
result: T,
): Omit<T, "screen"> {
const { screen: _screen, ...rest } = result as T & { screen?: unknown };
if (!Array.isArray(rest.elements)) return rest;
return {
...rest,
elements: rest.elements.map(({ box: _box, ...element }) => element),
};
}
function validScreen(value: unknown): ComputerScreenSize | undefined {
if (!value || typeof value !== "object") return undefined;
const { width, height } = value as Record<string, unknown>;
if (!isPixel(width) || !isPixel(height) || width === 0 || height === 0) return undefined;
return { width, height };
}
function validBox(value: unknown): value is BrowserScreenBox {
return (
Array.isArray(value) &&
value.length === 4 &&
value.every(isPixel) &&
(value[2] as number) > 0 &&
(value[3] as number) > 0
);
}
function isPixel(value: unknown): value is number {
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100_000;
}

View File

@ -25,7 +25,7 @@ import {
routineWakeupJob,
runContinueJob,
} from "@rakazo/adapter-kit";
import type { ComputerPointerPayload, ModelTier, RunStatus } from "@rakazo/contracts";
import type { ModelTier, RunStatus } from "@rakazo/contracts";
import {
ATTACHMENT_MAX_BYTES,
BOT_DESCRIPTION_MAX_LENGTH,
@ -45,7 +45,6 @@ import {
applyJudgeDecision,
assertTransition,
botMessageAllowsSilence,
computerScreenSize,
connectorKindFromToolName,
containsSecret,
createStreamingRedactor,
@ -166,7 +165,6 @@ import {
} from "./bot-secrets.js";
import { createBrowserProvider } from "./browser-provider-factory.js";
import {
type BrowserPointerHooks,
browserActFromTool,
browserNavigateFromTool,
browserSnapshotFromTool,
@ -195,11 +193,6 @@ import {
renewComputerExecutionLease,
screenLeaseIdForRun,
} from "./computer-lifecycle.js";
import {
BrowserPointerTargets,
computerPointerPayload,
computerPointerSteps,
} from "./computer-pointer.js";
import { withComputerScreenAvailability } from "./computer-screens.js";
import {
assertTeamShellCommandAllowed,
@ -1660,27 +1653,6 @@ export function createRunExecutor(deps: ExecutorDeps) {
lastComputerFrameId = observation.frameId;
return result;
};
// Best-effort: the visible cursor must never change a tool result or strand the run.
const announcePointer = async (payload: ComputerPointerPayload) => {
try {
await deps.events.append({
spaceId: run.spaceId,
threadId: thread.id,
botId: bot.id,
type: "thread.computer",
runId,
payload,
});
} catch (error) {
getLogger().warn("computer pointer append failed", {
error: sanitizeConnectorError(error, runSecrets),
});
}
};
const browserPointer: BrowserPointerHooks = {
targets: new BrowserPointerTargets(),
announce: announcePointer,
};
const pauseForApproval = () => {
approvalPausePending = true;
@ -2236,16 +2208,10 @@ export function createRunExecutor(deps: ExecutorDeps) {
}
workspaceCheckpoint.markDirty();
return computerScreenToolResult(async () => {
const actions = parseComputerActions(args.actions);
const pointer = computerPointerPayload(
computerScreenSize(computer.kind),
computerPointerSteps(actions),
);
if (pointer) await announcePointer(pointer);
const result = await deps.sandbox.act(
computer,
{
actions,
actions: parseComputerActions(args.actions),
observe: args.observe !== false,
settleMs: Number(args.settle_ms ?? 350),
},
@ -2629,10 +2595,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
: name === "browser_snapshot"
? browserSnapshotFromTool
: browserActFromTool;
return computerScreenToolResult(
() => tool(browser, computer, context, args, browserPointer),
finish,
);
return computerScreenToolResult(() => tool(browser, computer, context, args), finish);
}
if (name.startsWith("cloud_agent_")) {

View File

@ -1,60 +0,0 @@
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { listProjectWorkspaceFiles, readProjectWorkspaceFile } from "./project-workspace-files.js";
describe("project workspace files", () => {
const roots: string[] = [];
afterEach(() => {
for (const root of roots.splice(0)) {
rmSync(root, { recursive: true, force: true });
}
});
function linkedFolderBesideSecret() {
const base = mkdtempSync(path.join(tmpdir(), "project-workspace-files-"));
roots.push(base);
const project = path.join(base, "project");
const outside = path.join(base, "outside");
mkdirSync(path.join(project, "docs"), { recursive: true });
mkdirSync(outside);
writeFileSync(path.join(project, "docs", "readme.md"), "inside");
writeFileSync(path.join(outside, "secret.txt"), "outside");
return { project, outside };
}
it("lists and reads files inside the linked folder, including symlinks that stay inside", async () => {
const { project } = linkedFolderBesideSecret();
symlinkSync(path.join(project, "docs", "readme.md"), path.join(project, "readme-link.md"));
const listed = await listProjectWorkspaceFiles(project, "project");
expect(listed.map((entry) => entry.path).sort()).toEqual([
"project/docs",
"project/readme-link.md",
]);
expect(await listProjectWorkspaceFiles(project, "project/docs")).toEqual([
{ path: "project/docs/readme.md", kind: "file", size: 6 },
]);
const read = await readProjectWorkspaceFile(project, "project/readme-link.md");
expect(new TextDecoder().decode(read)).toBe("inside");
});
it("keeps symlinks from leaving the linked folder", async () => {
const { project, outside } = linkedFolderBesideSecret();
symlinkSync(outside, path.join(project, "escape"));
symlinkSync(path.join(outside, "secret.txt"), path.join(project, "secret-link.txt"));
symlinkSync(path.join(project, "missing"), path.join(project, "dangling"));
const listed = await listProjectWorkspaceFiles(project, "project");
expect(listed.map((entry) => entry.path)).toEqual(["project/docs"]);
await expect(listProjectWorkspaceFiles(project, "project/escape")).rejects.toThrow(/escapes/i);
await expect(readProjectWorkspaceFile(project, "project/escape/secret.txt")).rejects.toThrow(
/escapes/i,
);
await expect(readProjectWorkspaceFile(project, "project/secret-link.txt")).rejects.toThrow(
/escapes/i,
);
});
});

View File

@ -1,8 +1,6 @@
import { readdir, readFile, realpath, stat } from "node:fs/promises";
import path from "node:path";
import { readdir, readFile, stat } from "node:fs/promises";
import type { ComputerFileEntry } from "@rakazo/adapter-kit";
import { normalizeWorkspacePath } from "./computer-support.js";
import { isAllowedDesktopPath } from "./desktop-sandbox-paths.js";
import {
isProjectWorkspacePath,
projectWorkspaceRelativePath,
@ -16,34 +14,28 @@ export async function listProjectWorkspaceFiles(
if (!isProjectWorkspacePath(storedPath)) {
throw new Error("Path is not under the project workspace");
}
const root = await realpath(workspaceRoot);
const hostDirectory = await containedHostPath(
root,
resolveProjectHostPath(workspaceRoot, storedPath),
);
const hostDirectory = resolveProjectHostPath(workspaceRoot, storedPath);
const relativeDirectory = projectWorkspaceRelativePath(storedPath);
const entries = await readdir(hostDirectory, { withFileTypes: true });
const listed = await Promise.all(
entries.map(async (entry) => {
// A symlink that leaves the linked folder, or leads nowhere, is not part of the project.
const info = await containedHostPath(root, path.join(hostDirectory, entry.name))
.then((target) => stat(target))
.catch(() => null);
if (!info) return [];
const childHost = resolveProjectHostPath(
workspaceRoot,
relativeDirectory ? `project/${relativeDirectory}/${entry.name}` : `project/${entry.name}`,
);
const info = await stat(childHost);
const childStored = normalizeWorkspacePath(
relativeDirectory ? `project/${relativeDirectory}/${entry.name}` : `project/${entry.name}`,
);
return [
{
path: childStored,
kind: info.isDirectory() ? ("dir" as const) : ("file" as const),
size: info.size,
...(info.isFile() && info.mode & 0o100 ? { executable: true } : {}),
},
];
return {
path: childStored,
kind: info.isDirectory() ? ("dir" as const) : ("file" as const),
size: info.size,
...(info.isFile() && info.mode & 0o100 ? { executable: true } : {}),
};
}),
);
return listed.flat();
return listed;
}
export async function readProjectWorkspaceFile(
@ -51,22 +43,10 @@ export async function readProjectWorkspaceFile(
storedPath: string,
options?: { maxBytes?: number },
): Promise<Uint8Array> {
const hostPath = await containedHostPath(
await realpath(workspaceRoot),
resolveProjectHostPath(workspaceRoot, storedPath),
);
const hostPath = resolveProjectHostPath(workspaceRoot, storedPath);
const info = await stat(hostPath);
if (options?.maxBytes !== undefined && info.size > options.maxBytes) {
throw new Error(`computer file exceeds ${options.maxBytes} bytes`);
}
return new Uint8Array(await readFile(hostPath));
}
/** Follows symlinks, but only to places inside the linked folder. */
async function containedHostPath(root: string, candidate: string): Promise<string> {
const target = await realpath(candidate);
if (!isAllowedDesktopPath(target, [root])) {
throw new Error("Path escapes the project workspace");
}
return target;
}

View File

@ -323,11 +323,7 @@ export function inferScript(
{
name: "computer_act",
args: {
// Click into the focused field first so the visible cursor has a target.
actions: [
{ kind: "click", x: 640, y: 400 },
{ kind: "type", text: typed },
],
actions: [{ kind: "type", text: typed }],
observe: true,
},
},

View File

@ -1,38 +0,0 @@
import * as z from "zod";
export const ComputerPointerAction = z.enum([
"move",
"click",
"double_click",
"right_click",
"down",
"up",
"type",
"key",
"scroll",
]);
export type ComputerPointerAction = z.infer<typeof ComputerPointerAction>;
/** Screen pixels; absent for keyboard steps that act wherever the pointer already is. */
export const ComputerPointerStep = z.object({
action: ComputerPointerAction,
x: z.number().int().min(0).max(100_000).optional(),
y: z.number().int().min(0).max(100_000).optional(),
});
export type ComputerPointerStep = z.infer<typeof ComputerPointerStep>;
export const MAX_COMPUTER_POINTER_STEPS = 48;
/**
* Payload of a `thread.computer` event with `kind: "pointer"`: where a bot is about to point,
* click or type on its computer screen, so clients can draw its cursor over the live screen.
*/
export const ComputerPointerPayload = z.object({
kind: z.literal("pointer"),
screen: z.object({
width: z.number().int().positive().max(100_000),
height: z.number().int().positive().max(100_000),
}),
steps: z.array(ComputerPointerStep).min(1).max(MAX_COMPUTER_POINTER_STEPS),
});
export type ComputerPointerPayload = z.infer<typeof ComputerPointerPayload>;

View File

@ -38,39 +38,20 @@ export interface RakazoDesktopOAuthCallback {
export interface RakazoDesktopBrowserPreviewState {
url: string;
title: string;
loading: boolean;
canGoBack: boolean;
canGoForward: boolean;
/** The last main-frame load failure for `url`, cleared by the next successful load. */
error: string | null;
devServerRunning: boolean;
}
/** Rectangle in CSS pixels of the app window's content area. */
export interface RakazoDesktopBrowserBounds {
x: number;
y: number;
width: number;
height: number;
}
/**
* Optional desktop bridge for the in-app browser: a native Chromium view the main process lays
* over the panel the renderer reserves for it. Absent in the browser, where an iframe is used.
*/
/** Optional desktop bridge for the in-app browser preview panel. Absent in the browser. */
export interface RakazoDesktopBrowserPreview {
/** Shows the view over these bounds; call again whenever the reserved area moves. */
show: (bounds: RakazoDesktopBrowserBounds) => Promise<void>;
hide: () => Promise<void>;
/** Only http(s) URLs load; anything else is rejected. */
getState: () => Promise<RakazoDesktopBrowserPreviewState>;
navigate: (url: string) => Promise<void>;
back: () => Promise<void>;
forward: () => Promise<void>;
reload: () => Promise<void>;
stop: () => Promise<void>;
openDevTools: () => Promise<void>;
state: () => Promise<RakazoDesktopBrowserPreviewState>;
onState: (listener: (state: RakazoDesktopBrowserPreviewState) => void) => () => void;
onNavigate: (listener: (url: string) => void) => () => void;
onLoadingProgress: (listener: (progress: number) => void) => () => void;
devServer: {
start: () => Promise<void>;
stop: () => Promise<void>;
};
}
export interface RakazoDesktop {

View File

@ -1,7 +1,6 @@
export * from "./ai-consent.js";
export * from "./attachments.js";
export * from "./bot-secrets.js";
export * from "./computer-pointer.js";
export * from "./desktop.js";
export * from "./domain.js";
export * from "./events.js";

View File

@ -1,10 +1,19 @@
// Server-only. Kept out of the package index so browser bundles never load `node:child_process`.
import { spawn } from "node:child_process";
import { access } from "node:fs/promises";
import path from "node:path";
import { gitStatusArgv, gitUntrackedFilesArgv, parseGitNameOnly } from "./compose-update.js";
import { DEFAULT_UPDATE_REMOTE, parseGitStatusPorcelain } from "./self-update.js";
// Lazy import to avoid bundling `node:child_process` in browser bundles.
// `defaultGitRunner` is server-only; lazy-loading ensures tree-shaking works.
type SpawnFn = typeof import("node:child_process").spawn;
let _spawn: SpawnFn | undefined;
const getSpawn = (): SpawnFn => {
if (_spawn === undefined) {
_spawn = require("node:child_process").spawn as SpawnFn;
}
return _spawn;
};
export interface GitCheckoutStatus {
present: boolean;
remoteUrl: string | null;
@ -23,6 +32,7 @@ export type GitRunner = (args: string[], cwd: string) => Promise<GitRunnerResult
export async function defaultGitRunner(args: string[], cwd: string): Promise<GitRunnerResult> {
return new Promise((resolve) => {
const spawn = getSpawn();
const child = spawn("git", args, { cwd, env: process.env });
let output = "";
child.stdout.on("data", (chunk: Buffer | string) => {

View File

@ -3,7 +3,6 @@ import {
computerInputForDomKey,
computerScreenSize,
mapTeachPointer,
screenPointInViewer,
teachCaptureKey,
} from "./teach-recording.js";
@ -89,21 +88,3 @@ describe("mapTeachPointer", () => {
});
});
});
describe("screenPointInViewer", () => {
it("projects through the same letterbox the viewer draws", () => {
const screen = { width: 1280, height: 800 };
// A wide viewer pillarboxes the 16:10 screen.
expect(screenPointInViewer({ x: 0, y: 0 }, { width: 1000, height: 400 }, screen)).toEqual({
x: 180,
y: 0,
});
expect(screenPointInViewer({ x: 1280, y: 800 }, { width: 1000, height: 400 }, screen)).toEqual({
x: 820,
y: 400,
});
const rect = { left: 0, top: 0, width: 640, height: 600 };
const projected = screenPointInViewer({ x: 640, y: 400 }, rect, screen);
expect(mapTeachPointer(projected.x, projected.y, rect, screen)).toEqual({ x: 640, y: 400 });
});
});

View File

@ -91,18 +91,3 @@ export function mapTeachPointer(
y: clamp(Math.round((clientY - rect.top - offsetY) / scale), screen.height - 1),
};
}
/** Inverse of {@link mapTeachPointer}: where a screen pixel lands inside the letterboxed viewer. */
export function screenPointInViewer(
point: { x: number; y: number },
viewer: { width: number; height: number },
screen: { width: number; height: number },
): { x: number; y: number } {
const scale = Math.min(viewer.width / screen.width, viewer.height / screen.height) || 1;
const offsetX = (viewer.width - screen.width * scale) / 2;
const offsetY = (viewer.height - screen.height * scale) / 2;
return {
x: offsetX + clamp(point.x, screen.width) * scale,
y: offsetY + clamp(point.y, screen.height) * scale,
};
}

View File

@ -1,12 +1,12 @@
import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs";
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { PrismaClient } from "./client.js";
import {
assertProjectWorkspaceDirectory,
InvalidProjectWorkspacePathError,
normalizeProjectWorkspacePath,
resolveProjectWorkspaceDirectory,
wireProjectWorkspace,
} from "./project-workspace.js";
@ -43,21 +43,11 @@ describe("project workspace wiring", () => {
it("rejects missing folders", async () => {
const missing = path.join(tmpdir(), `missing-${Date.now()}`);
await expect(resolveProjectWorkspaceDirectory(missing)).rejects.toBeInstanceOf(
await expect(assertProjectWorkspaceDirectory(missing)).rejects.toBeInstanceOf(
InvalidProjectWorkspacePathError,
);
});
it("resolves a symlinked folder to the real folder behind it", async () => {
const root = tempDir("project-workspace-symlink-");
const real = path.join(root, "real");
mkdirSync(real);
symlinkSync(real, path.join(root, "link"));
expect(await resolveProjectWorkspaceDirectory(path.join(root, "link"))).toBe(
realpathSync(real),
);
});
it("persists workspace and git metadata on the space", async () => {
const root = tempDir("project-workspace-wire-");
mkdirSync(path.join(root, ".git"));
@ -72,9 +62,9 @@ describe("project workspace wiring", () => {
const wired = await wireProjectWorkspace(prisma as unknown as Pick<PrismaClient, "space">, {
spaceId: "space-1",
workspacePath: await resolveProjectWorkspaceDirectory(root),
workspacePath: root,
});
expect(wired.path).toBe(realpathSync(root));
expect(wired.path).toBe(path.resolve(root));
expect(wired.git).toEqual({
remoteUrl: "https://github.com/example/demo.git",
defaultBranch: "main",
@ -83,7 +73,7 @@ describe("project workspace wiring", () => {
dirtyPaths: [],
});
expect(updates[0]).toMatchObject({
workspacePath: realpathSync(root),
workspacePath: path.resolve(root),
gitRemoteUrl: "https://github.com/example/demo.git",
gitDefaultBranch: "main",
gitHeadCommit: "abc123",

View File

@ -1,4 +1,4 @@
import { realpath, stat } from "node:fs/promises";
import { access, stat } from "node:fs/promises";
import path from "node:path";
import { readGitCheckout } from "@rakazo/core/git-checkout";
import type { PrismaClient } from "./client.js";
@ -22,13 +22,11 @@ export function normalizeProjectWorkspacePath(input: string): string {
return path.resolve(trimmed);
}
/** The real folder behind a path, so a symlink cannot later point the link somewhere else. */
export async function resolveProjectWorkspaceDirectory(workspacePath: string): Promise<string> {
const normalized = normalizeProjectWorkspacePath(workspacePath);
let resolved: string;
export async function assertProjectWorkspaceDirectory(workspacePath: string): Promise<void> {
const resolved = normalizeProjectWorkspacePath(workspacePath);
let info: Awaited<ReturnType<typeof stat>>;
try {
resolved = await realpath(normalized);
await access(resolved);
info = await stat(resolved);
} catch {
throw new InvalidProjectWorkspacePathError("That folder does not exist or is not readable.");
@ -36,7 +34,6 @@ export async function resolveProjectWorkspaceDirectory(workspacePath: string): P
if (!info.isDirectory()) {
throw new InvalidProjectWorkspacePathError("That path is not a folder.");
}
return resolved;
}
export interface WiredProjectWorkspace {
@ -50,17 +47,19 @@ export interface WiredProjectWorkspace {
} | null;
}
/** Persist a folder from `resolveProjectWorkspaceDirectory`, and its git metadata, on a new space. */
/** Persist a host folder and optional git metadata on a newly created space. */
export async function wireProjectWorkspace(
prisma: Pick<PrismaClient, "space">,
input: { spaceId: string; workspacePath: string },
): Promise<WiredProjectWorkspace> {
const git = await readGitCheckout(input.workspacePath);
const resolved = normalizeProjectWorkspacePath(input.workspacePath);
await assertProjectWorkspaceDirectory(resolved);
const git = await readGitCheckout(resolved);
await prisma.space.update({
where: { id: input.spaceId },
data: {
workspacePath: input.workspacePath,
workspacePath: resolved,
gitRemoteUrl: git.present && git.remoteUrl ? git.remoteUrl : null,
gitDefaultBranch: git.present && git.defaultBranch ? git.defaultBranch : null,
gitHeadCommit: git.present ? git.headCommit : null,
@ -68,7 +67,7 @@ export async function wireProjectWorkspace(
});
return {
path: input.workspacePath,
path: resolved,
git:
git.present && git.remoteUrl && git.defaultBranch
? {

View File

@ -1,4 +1,4 @@
import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { ComposioEmulator, LocalAgentHomeStore, teamBotWorkspaceDirectory } from "@rakazo/adapters";
@ -2185,110 +2185,6 @@ describeWithDatabase("API authorization and resource isolation", () => {
});
}
});
it("lets only the deployment owner link a host folder and read its files", async () => {
const owner = await signup(app, `folder-owner-${stamp}@rakazo.test`, "Folder Owner");
const member = await signup(app, `folder-member-${stamp}@rakazo.test`, "Folder Member");
const ownerActor = await rpc<Actor>(app, owner, "me");
const settings = await handles.prisma.deploymentSettings.findUniqueOrThrow({
where: { id: "default" },
select: { ownerUserId: true },
});
await handles.prisma.deploymentSettings.update({
where: { id: "default" },
data: { ownerUserId: ownerActor.userId },
});
// Stands in for any folder the API process can read, such as its own checkout.
const hostFolder = realpathSync(mkdtempSync(path.join(tmpdir(), "rakazo-authz-host-")));
writeFileSync(path.join(hostFolder, ".env"), "HOST_SECRET=not-a-real-secret\n");
try {
await expectForbidden(app, member, "spaces/create", {
name: "Member folder",
workspacePath: hostFolder,
});
const memberNavigation = await rpc<SpaceNavigation>(app, member, "spaces/list");
expect(memberNavigation.spaces.map((space) => space.name)).not.toContain("Member folder");
// A space linked before only the owner could link stays closed to the member.
const legacy = await rpc<Space>(app, member, "spaces/create", { name: "Legacy folder" });
await handles.prisma.space.update({
where: { id: legacy.id },
data: { workspacePath: hostFolder },
});
const memberBot = await rpc<Bot>(
app,
member,
"bots/create",
botInput("Member Folder Reader"),
legacy.id,
);
await expectForbidden(
app,
member,
"computer/files",
{ botId: memberBot.id, path: "project" },
legacy.id,
);
await expectForbidden(
app,
member,
"computer/readFile",
{ botId: memberBot.id, path: "project/.env" },
legacy.id,
);
const legacyNavigation = await rpc<SpaceNavigation>(app, member, "spaces/list");
expect(legacyNavigation.spaces.find((space) => space.id === legacy.id)?.workspace).toBeNull();
// A bad folder is rejected before the project exists, so fixing it and retrying leaves one.
for (const badPath of [path.join(hostFolder, "missing"), path.join(hostFolder, ".env")]) {
const rejected = await raw(app, owner, "spaces/create", {
name: "Owner folder",
workspacePath: badPath,
});
expect(rejected.status, badPath).toBe(400);
}
const linked = await rpc<Space>(app, owner, "spaces/create", {
name: "Owner folder",
workspacePath: hostFolder,
});
expect(linked.workspace?.path).toBe(hostFolder);
const ownerNavigation = await rpc<SpaceNavigation>(app, owner, "spaces/list");
expect(
ownerNavigation.spaces
.filter((space) => space.name === "Owner folder")
.map((space) => space.workspace?.path),
).toEqual([hostFolder]);
const ownerBot = await rpc<Bot>(
app,
owner,
"bots/create",
botInput("Owner Folder Reader"),
linked.id,
);
const listing = await rpc<Array<{ path: string }>>(
app,
owner,
"computer/files",
{ botId: ownerBot.id, path: "project" },
linked.id,
);
expect(listing.map((entry) => entry.path)).toContain("project/.env");
const file = await rpc<{ content: string }>(
app,
owner,
"computer/readFile",
{ botId: ownerBot.id, path: "project/.env" },
linked.id,
);
expect(file.content).toContain("HOST_SECRET");
} finally {
await handles.prisma.deploymentSettings.update({
where: { id: "default" },
data: { ownerUserId: settings.ownerUserId },
});
rmSync(hostFolder, { recursive: true, force: true });
}
});
});
function botInput(name: string) {