Compare commits
11 Commits
claude/ama
...
cursor/ci-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1dd4ff4ba1 | ||
|
|
1c3d8c40fe | ||
|
|
80c56eef36 | ||
|
|
d912712668 | ||
|
|
e28c3acf02 | ||
|
|
1704d1cec9 | ||
|
|
a15da068d7 | ||
|
|
49431def83 | ||
|
|
3c3961f242 | ||
|
|
2b967df569 | ||
|
|
f8eabee997 |
28
.github/workflows/ci.yml
vendored
28
.github/workflows/ci.yml
vendored
@ -246,6 +246,7 @@ jobs:
|
||||
publish-mobile-update:
|
||||
name: Publish compatible mobile update
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
continue-on-error: true
|
||||
needs: [lint, check, build, test, test-integration, test-e2e]
|
||||
runs-on: [self-hosted, evoo-ci, Linux]
|
||||
timeout-minutes: 20
|
||||
@ -344,23 +345,30 @@ jobs:
|
||||
fi
|
||||
echo "publish=$publish" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Require Expo automation token
|
||||
- name: Check Expo automation token
|
||||
id: expo_token
|
||||
if: steps.ota.outputs.publish == 'true'
|
||||
env:
|
||||
EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }}
|
||||
run: |
|
||||
if [[ -z "$EXPO_TOKEN" ]]; then
|
||||
echo "Add an EXPO_TOKEN repository secret with access to the inbox-zero/rakazo Expo project." >&2
|
||||
exit 1
|
||||
{
|
||||
echo "### Mobile OTA skipped"
|
||||
echo
|
||||
echo "No \`EXPO_TOKEN\` repository secret is configured, so CI will not publish a production update. Add a token with access to the inbox-zero/rakazo Expo project when OTA from main is desired."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "ready=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "ready=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: pnpm/action-setup@f40ffcd9367d9f12939873eb1018b921a783ffaa # v4
|
||||
if: steps.ota.outputs.publish == 'true'
|
||||
if: steps.ota.outputs.publish == 'true' && steps.expo_token.outputs.ready == 'true'
|
||||
id: pnpm-setup
|
||||
with:
|
||||
dest: ${{ runner.temp }}/setup-pnpm-${{ github.run_id }}-${{ github.job }}
|
||||
- name: Isolate pnpm store and PATH
|
||||
if: steps.ota.outputs.publish == 'true'
|
||||
if: steps.ota.outputs.publish == 'true' && steps.expo_token.outputs.ready == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@ -377,25 +385,25 @@ jobs:
|
||||
mkdir -p "$store"
|
||||
echo "npm_config_store_dir=${store}" >> "$GITHUB_ENV"
|
||||
- uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
|
||||
if: steps.ota.outputs.publish == 'true'
|
||||
if: steps.ota.outputs.publish == 'true' && steps.expo_token.outputs.ready == 'true'
|
||||
with:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
- name: Set up Expo
|
||||
if: steps.ota.outputs.publish == 'true'
|
||||
if: steps.ota.outputs.publish == 'true' && steps.expo_token.outputs.ready == 'true'
|
||||
uses: expo/expo-github-action@eab7a230208c952974db8c3245cfd78402c7b385 # v9
|
||||
with:
|
||||
eas-version: latest
|
||||
packager: pnpm
|
||||
token: ${{ secrets.EXPO_TOKEN }}
|
||||
- name: Install dependencies
|
||||
if: steps.ota.outputs.publish == 'true'
|
||||
if: steps.ota.outputs.publish == 'true' && steps.expo_token.outputs.ready == 'true'
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Verify mobile project
|
||||
if: steps.ota.outputs.publish == 'true'
|
||||
if: steps.ota.outputs.publish == 'true' && steps.expo_token.outputs.ready == 'true'
|
||||
run: pnpm --filter @rakazo/mobile check
|
||||
- name: Publish production update
|
||||
if: steps.ota.outputs.publish == 'true'
|
||||
if: steps.ota.outputs.publish == 'true' && steps.expo_token.outputs.ready == 'true'
|
||||
working-directory: apps/mobile
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
4
.github/workflows/publish-server-image.yml
vendored
4
.github/workflows/publish-server-image.yml
vendored
@ -263,8 +263,8 @@ jobs:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
owner_type="$(gh api "repos/${GITHUB_REPOSITORY}" --jq -r '.owner.type')"
|
||||
private="$(gh api "repos/${GITHUB_REPOSITORY}" --jq -r '.private')"
|
||||
owner_type="$(gh api "repos/${GITHUB_REPOSITORY}" --jq '.owner.type')"
|
||||
private="$(gh api "repos/${GITHUB_REPOSITORY}" --jq '.private')"
|
||||
if [[ "$owner_type" == "User" && "$private" == "true" ]]; then
|
||||
echo "enabled=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Skipping registry attestation: unavailable for private repositories on personal accounts."
|
||||
|
||||
@ -1,11 +1,42 @@
|
||||
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, realpath, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type * as db from "@rakazo/db";
|
||||
import { SpaceLimitError } from "@rakazo/db";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { getScanStatus, startOnboardingScan } from "./onboarding-scan.js";
|
||||
|
||||
const owner = { userId: "user-owner", isDeploymentOwner: true };
|
||||
const member = { userId: "user-member", isDeploymentOwner: false };
|
||||
const createSpaceForMember = vi.hoisted(() => vi.fn());
|
||||
vi.mock("@rakazo/db", async (original) => ({
|
||||
...(await original<typeof db>()),
|
||||
createSpaceForMember,
|
||||
}));
|
||||
|
||||
const owner = { userId: "user-owner", spaceId: "space-owner", isDeploymentOwner: true };
|
||||
const member = { userId: "user-member", spaceId: "space-member", isDeploymentOwner: false };
|
||||
|
||||
type SpaceUpdate = { where: { id: string }; data: { workspacePath: string } };
|
||||
|
||||
/** The spaces the scan reads and links; new ones come from `createSpaceForMember`. */
|
||||
function fakeDatabase({ spaceLimit = Number.POSITIVE_INFINITY } = {}) {
|
||||
const spaces: Array<{ id: string; name: string; workspacePath: string | null }> = [];
|
||||
createSpaceForMember.mockImplementation(async (_prisma: unknown, input: { name: string }) => {
|
||||
if (spaces.length >= spaceLimit) throw new SpaceLimitError();
|
||||
const space = { id: `space-${spaces.length + 1}`, name: input.name, workspacePath: null };
|
||||
spaces.push(space);
|
||||
return { id: space.id, name: space.name };
|
||||
});
|
||||
const prisma = {
|
||||
space: {
|
||||
findMany: async () => spaces.filter((space) => space.workspacePath),
|
||||
update: async ({ where, data }: SpaceUpdate) => {
|
||||
const space = spaces.find((candidate) => candidate.id === where.id);
|
||||
if (space) space.workspacePath = data.workspacePath;
|
||||
},
|
||||
},
|
||||
} as unknown as db.PrismaClient;
|
||||
return { prisma, spaces };
|
||||
}
|
||||
|
||||
async function scanHomeWithOneFile(): Promise<string> {
|
||||
const home = await mkdtemp(join(tmpdir(), "evoo-scan-"));
|
||||
@ -14,6 +45,12 @@ async function scanHomeWithOneFile(): Promise<string> {
|
||||
return home;
|
||||
}
|
||||
|
||||
async function scanHomeWithRepos(...repos: string[]): Promise<string> {
|
||||
const home = await scanHomeWithOneFile();
|
||||
for (const repo of repos) await mkdir(join(home, repo, ".git"), { recursive: true });
|
||||
return home;
|
||||
}
|
||||
|
||||
async function completed(scanId: string, userId: string) {
|
||||
return await vi.waitFor(
|
||||
async () => {
|
||||
@ -25,13 +62,21 @@ async function completed(scanId: string, userId: string) {
|
||||
);
|
||||
}
|
||||
|
||||
/** The emulated runtime that the shared test setup pins keeps the scan simulated. */
|
||||
function allowHostScan() {
|
||||
vi.stubEnv("AGENT_RUNTIME", "");
|
||||
vi.stubEnv("SANDBOX_PROVIDER", "");
|
||||
}
|
||||
|
||||
describe("onboarding scan", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
createSpaceForMember.mockReset();
|
||||
});
|
||||
|
||||
it("hides a scan from every account except the one that started it", async () => {
|
||||
const { scanId } = await startOnboardingScan({ actor: member }, {});
|
||||
const { prisma } = fakeDatabase();
|
||||
const { scanId } = await startOnboardingScan({ actor: member }, prisma);
|
||||
await expect(getScanStatus(scanId, owner.userId)).rejects.toThrow(/Scan not found/);
|
||||
await expect(getScanStatus(scanId, member.userId)).resolves.toMatchObject({
|
||||
status: expect.any(String),
|
||||
@ -40,25 +85,77 @@ describe("onboarding scan", () => {
|
||||
});
|
||||
|
||||
it("never reads the host without a configured scan home", async () => {
|
||||
const { scanId } = await startOnboardingScan({ actor: owner }, {});
|
||||
const { prisma } = fakeDatabase();
|
||||
const { scanId } = await startOnboardingScan({ actor: owner }, prisma);
|
||||
const status = await completed(scanId, owner.userId);
|
||||
expect(status.status).toBe("complete");
|
||||
expect(status.result).toMatchObject({ localRepos: [], fileSummary: { totalFiles: 0 } });
|
||||
});
|
||||
|
||||
it("lets only the deployment owner discover the configured scan home", async () => {
|
||||
// The shared test setup pins the emulated runtime, which also keeps the scan simulated.
|
||||
vi.stubEnv("AGENT_RUNTIME", "");
|
||||
vi.stubEnv("SANDBOX_PROVIDER", "");
|
||||
const scanHome = await scanHomeWithOneFile();
|
||||
allowHostScan();
|
||||
const scanHome = await scanHomeWithRepos("Projects/app");
|
||||
const { prisma } = fakeDatabase();
|
||||
|
||||
const asMember = await startOnboardingScan({ actor: member, scanHome }, {});
|
||||
const asMember = await startOnboardingScan({ actor: member, scanHome }, prisma);
|
||||
const memberStatus = await completed(asMember.scanId, member.userId);
|
||||
expect(memberStatus.result).toMatchObject({ fileSummary: { totalFiles: 0 } });
|
||||
expect(createSpaceForMember).not.toHaveBeenCalled();
|
||||
|
||||
const asOwner = await startOnboardingScan({ actor: owner, scanHome }, {});
|
||||
const asOwner = await startOnboardingScan({ actor: owner, scanHome }, prisma);
|
||||
const ownerStatus = await completed(asOwner.scanId, owner.userId);
|
||||
expect(ownerStatus.status).toBe("complete");
|
||||
expect(ownerStatus.result).toMatchObject({ fileSummary: { totalFiles: 1 } });
|
||||
});
|
||||
|
||||
it("links each repository of the deployment owner to one project of its own", async () => {
|
||||
allowHostScan();
|
||||
const scanHome = await scanHomeWithRepos("Projects/app", ".config/nvim");
|
||||
const app = await realpath(join(scanHome, "Projects", "app"));
|
||||
const { prisma, spaces } = fakeDatabase();
|
||||
|
||||
const first = await startOnboardingScan({ actor: owner, scanHome }, prisma);
|
||||
const firstStatus = await completed(first.scanId, owner.userId);
|
||||
expect(firstStatus.status).toBe("complete");
|
||||
expect(firstStatus.result).toMatchObject({
|
||||
localRepos: [{ name: "app" }],
|
||||
createdProjects: [{ spaceId: "space-1", name: "app", path: app }],
|
||||
});
|
||||
expect(createSpaceForMember).toHaveBeenCalledWith(prisma, {
|
||||
currentSpaceId: owner.spaceId,
|
||||
userId: owner.userId,
|
||||
name: "app",
|
||||
});
|
||||
expect(spaces).toEqual([{ id: "space-1", name: "app", workspacePath: app }]);
|
||||
|
||||
// Scanning again finds the linked project instead of creating a second one.
|
||||
const again = await startOnboardingScan({ actor: owner, scanHome }, prisma);
|
||||
const againStatus = await completed(again.scanId, owner.userId);
|
||||
expect(againStatus.result).toMatchObject({ createdProjects: [] });
|
||||
expect(createSpaceForMember).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("joins a running scan instead of linking its repositories twice", async () => {
|
||||
allowHostScan();
|
||||
const scanHome = await scanHomeWithRepos("Projects/app");
|
||||
const { prisma, spaces } = fakeDatabase();
|
||||
|
||||
const first = await startOnboardingScan({ actor: owner, scanHome }, prisma);
|
||||
const second = await startOnboardingScan({ actor: owner, scanHome }, prisma);
|
||||
expect(second.scanId).toBe(first.scanId);
|
||||
await completed(first.scanId, owner.userId);
|
||||
expect(spaces).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("stops creating projects at the space limit and still completes", async () => {
|
||||
allowHostScan();
|
||||
const scanHome = await scanHomeWithRepos("Projects/a", "Projects/b", "Projects/c");
|
||||
const { prisma } = fakeDatabase({ spaceLimit: 1 });
|
||||
|
||||
const { scanId } = await startOnboardingScan({ actor: owner, scanHome }, prisma);
|
||||
const status = await completed(scanId, owner.userId);
|
||||
expect(status.status).toBe("complete");
|
||||
expect(status.result).toMatchObject({ createdProjects: [{ spaceId: "space-1" }] });
|
||||
expect(createSpaceForMember).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
@ -2,8 +2,16 @@
|
||||
// Runs synchronously on the host machine (no sandbox needed).
|
||||
import { spawn } from "node:child_process";
|
||||
import { access, readdir, stat } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import type { Actor } from "@rakazo/contracts";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import {
|
||||
createSpaceForMember,
|
||||
InvalidSpaceNameError,
|
||||
resolveProjectWorkspaceDirectory,
|
||||
SpaceLimitError,
|
||||
wireProjectWorkspace,
|
||||
} from "@rakazo/db";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@ -54,10 +62,9 @@ interface ProfileSummary {
|
||||
}
|
||||
|
||||
interface CreatedProject {
|
||||
id: string;
|
||||
name: string;
|
||||
spaceId: string;
|
||||
description: string;
|
||||
name: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
interface ScanRecord {
|
||||
@ -115,8 +122,10 @@ async function runSimulatedScan(record: ScanRecord) {
|
||||
record.completedAt = new Date();
|
||||
}
|
||||
|
||||
type ScanActor = Pick<Actor, "userId" | "spaceId" | "isDeploymentOwner">;
|
||||
|
||||
export interface OnboardingScanRequest {
|
||||
actor: Pick<Actor, "userId" | "isDeploymentOwner">;
|
||||
actor: ScanActor;
|
||||
botId?: string;
|
||||
/** Home-style directory to discover. Unset keeps the scan off the host filesystem. */
|
||||
scanHome?: string;
|
||||
@ -124,8 +133,12 @@ export interface OnboardingScanRequest {
|
||||
|
||||
export async function startOnboardingScan(
|
||||
{ actor, botId, scanHome }: OnboardingScanRequest,
|
||||
prisma: any,
|
||||
prisma: PrismaClient,
|
||||
): Promise<{ scanId: string }> {
|
||||
// A repeated start (a remount, a retry) joins the running scan, so no repository is linked twice.
|
||||
for (const running of scanRegistry.values()) {
|
||||
if (running.ownerUserId === actor.userId && !running.completedAt) return { scanId: running.id };
|
||||
}
|
||||
const scanId = `scan_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const record: ScanRecord = {
|
||||
id: scanId,
|
||||
@ -154,7 +167,7 @@ export async function startOnboardingScan(
|
||||
await runSimulatedScan(record);
|
||||
return;
|
||||
}
|
||||
const result = await executeFullScan(scanHome, botId ?? "", prisma);
|
||||
const result = await executeFullScan(scanHome, actor, prisma);
|
||||
record.status = "complete";
|
||||
record.progress = 100;
|
||||
record.message = "Scan complete — profile ready";
|
||||
@ -189,7 +202,11 @@ export async function getScanStatus(
|
||||
// Full scan pipeline
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function executeFullScan(homeDir: string, _botId: string, prisma: any): Promise<ScanResult> {
|
||||
async function executeFullScan(
|
||||
homeDir: string,
|
||||
actor: ScanActor,
|
||||
prisma: PrismaClient,
|
||||
): Promise<ScanResult> {
|
||||
// Phase 1: Local file scan
|
||||
const [fileSummary, projectFolders] = await scanFiles(homeDir);
|
||||
|
||||
@ -200,8 +217,8 @@ async function executeFullScan(homeDir: string, _botId: string, prisma: any): Pr
|
||||
// Phase 4: Profile compilation from findings
|
||||
const profileSummary = compileProfile(localRepos, fileSummary, projectFolders);
|
||||
|
||||
// Phase 6: Project folder creation
|
||||
const createdProjects = await createProjectFolders(localRepos, prisma);
|
||||
// Phase 6: Projects for discovered repositories
|
||||
const createdProjects = await createProjects(localRepos, actor, prisma);
|
||||
|
||||
// Phase 7: Sub-agent role creation
|
||||
const subAgentRoles = generateSubAgentRoles(profileSummary, localRepos);
|
||||
@ -413,9 +430,9 @@ async function findGitDirectories(dir: string, depth: number = 0): Promise<strin
|
||||
if (entry.name === ".git") {
|
||||
results.push(dirname(fullPath));
|
||||
} else if (
|
||||
// Dot folders hold tools and caches (shell and editor plugins), not the person's projects.
|
||||
!entry.name.startsWith(".") &&
|
||||
entry.name !== "node_modules" &&
|
||||
entry.name !== ".git" &&
|
||||
entry.name !== ".venv" &&
|
||||
entry.name !== "dist"
|
||||
) {
|
||||
results.push(...(await findGitDirectories(fullPath, depth + 1)));
|
||||
@ -622,38 +639,44 @@ function generateBriefing(
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 5: Project folder creation
|
||||
// Phase 5: Projects for discovered repositories
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function createProjectFolders(repos: RepoInfo[], prisma: any): Promise<CreatedProject[]> {
|
||||
/** Give each discovered repository a project linked to its folder, unless one already links it. */
|
||||
async function createProjects(
|
||||
repos: RepoInfo[],
|
||||
actor: ScanActor,
|
||||
prisma: PrismaClient,
|
||||
): Promise<CreatedProject[]> {
|
||||
const linked = await prisma.space.findMany({
|
||||
where: { workspacePath: { not: null }, memberships: { some: { userId: actor.userId } } },
|
||||
select: { workspacePath: true },
|
||||
});
|
||||
const linkedFolders = new Set(linked.map((space) => space.workspacePath));
|
||||
const created: CreatedProject[] = [];
|
||||
|
||||
for (const repo of repos) {
|
||||
// Compare and link the real folder, so the next scan recognizes the projects of this one.
|
||||
// Checking it before the project exists means a folder that cannot be linked leaves none.
|
||||
const folder = await resolveProjectWorkspaceDirectory(repo.path).catch(() => null);
|
||||
if (!folder || linkedFolders.has(folder)) continue;
|
||||
let space: { id: string; name: string };
|
||||
try {
|
||||
// Check if space already exists
|
||||
const existing = await prisma.space.findFirst({ where: { title: repo.name } });
|
||||
if (existing) {
|
||||
created.push({
|
||||
id: existing.id,
|
||||
name: repo.name,
|
||||
spaceId: existing.id,
|
||||
description: repo.remote || repo.path,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
// Create new space
|
||||
const space = await prisma.space.create({
|
||||
data: { title: repo.name, path: repo.path },
|
||||
space = await createSpaceForMember(prisma, {
|
||||
currentSpaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
name: basename(folder),
|
||||
});
|
||||
created.push({
|
||||
id: space.id,
|
||||
name: repo.name,
|
||||
spaceId: space.id,
|
||||
description: repo.remote || repo.path,
|
||||
});
|
||||
} catch {
|
||||
// Space might already exist or DB unavailable — skip
|
||||
} catch (error) {
|
||||
if (error instanceof SpaceLimitError) break;
|
||||
if (error instanceof InvalidSpaceNameError) continue;
|
||||
throw error;
|
||||
}
|
||||
const workspace = await wireProjectWorkspace(prisma, {
|
||||
spaceId: space.id,
|
||||
workspacePath: folder,
|
||||
});
|
||||
created.push({ spaceId: space.id, name: space.name, path: workspace.path });
|
||||
}
|
||||
|
||||
return created;
|
||||
|
||||
@ -137,6 +137,7 @@ import {
|
||||
releaseSpaceDeletionClaim,
|
||||
renewSpaceDeletionClaim,
|
||||
requireMembership,
|
||||
resolveProjectWorkspaceDirectory,
|
||||
SPACE_DELETION_CLAIM_TIMEOUT_MS,
|
||||
SpaceDeletionInProgressError,
|
||||
SpaceLimitError,
|
||||
@ -569,6 +570,22 @@ export function createRouter(deps: RouterDeps) {
|
||||
spaceNavigationDto(deps, context.actor, repos, groupRepos),
|
||||
),
|
||||
create: authed.spaces.create.handler(async ({ context, input }) => {
|
||||
// A linked folder lives on the host, and the host is the deployment owner's machine.
|
||||
if (input.workspacePath && !context.actor.isDeploymentOwner) {
|
||||
throw new ORPCError("FORBIDDEN");
|
||||
}
|
||||
// Check the folder before the space exists, so a bad path leaves no space behind.
|
||||
let workspacePath: string | null = null;
|
||||
if (input.workspacePath) {
|
||||
try {
|
||||
workspacePath = await resolveProjectWorkspaceDirectory(input.workspacePath);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidProjectWorkspacePathError) {
|
||||
throw new ORPCError("BAD_REQUEST", { message: error.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
let space: { id: string; name: string };
|
||||
try {
|
||||
space = await createSpaceForMember(deps.prisma, {
|
||||
@ -582,20 +599,9 @@ export function createRouter(deps: RouterDeps) {
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
let workspace = null;
|
||||
if (input.workspacePath) {
|
||||
try {
|
||||
workspace = await wireProjectWorkspace(deps.prisma, {
|
||||
spaceId: space.id,
|
||||
workspacePath: input.workspacePath,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidProjectWorkspacePathError) {
|
||||
throw new ORPCError("BAD_REQUEST", { message: error.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const workspace = workspacePath
|
||||
? await wireProjectWorkspace(deps.prisma, { spaceId: space.id, workspacePath })
|
||||
: null;
|
||||
return {
|
||||
id: space.id,
|
||||
name: space.name,
|
||||
@ -2230,10 +2236,7 @@ export function createRouter(deps: RouterDeps) {
|
||||
}
|
||||
}
|
||||
if (isProjectWorkspacePath(storedPath)) {
|
||||
const workspaceRoot = await requireProjectWorkspaceRoot(
|
||||
deps.prisma,
|
||||
context.actor.spaceId,
|
||||
);
|
||||
const workspaceRoot = await requireProjectWorkspaceRoot(deps.prisma, context.actor);
|
||||
const entries = await listProjectWorkspaceFiles(workspaceRoot, storedPath);
|
||||
return entries.map((entry) => ({
|
||||
...entry,
|
||||
@ -2281,10 +2284,7 @@ export function createRouter(deps: RouterDeps) {
|
||||
}
|
||||
}
|
||||
if (isProjectWorkspacePath(storedPath)) {
|
||||
const workspaceRoot = await requireProjectWorkspaceRoot(
|
||||
deps.prisma,
|
||||
context.actor.spaceId,
|
||||
);
|
||||
const workspaceRoot = await requireProjectWorkspaceRoot(deps.prisma, context.actor);
|
||||
const bytes = await readProjectWorkspaceFile(workspaceRoot, storedPath, {
|
||||
maxBytes: MAX_COMPUTER_TEXT_FILE_BYTES,
|
||||
});
|
||||
@ -4894,10 +4894,13 @@ function mapUpdaterError(error: unknown): never {
|
||||
|
||||
async function requireProjectWorkspaceRoot(
|
||||
prisma: Pick<PrismaClient, "space">,
|
||||
spaceId: string,
|
||||
actor: Actor,
|
||||
): Promise<string> {
|
||||
// Only the deployment owner may link a host folder, so only they may browse one. This also
|
||||
// closes folders that other members linked before linking was limited to the owner.
|
||||
if (!actor.isDeploymentOwner) throw new ORPCError("FORBIDDEN");
|
||||
const space = await prisma.space.findUnique({
|
||||
where: { id: spaceId },
|
||||
where: { id: actor.spaceId },
|
||||
select: { workspacePath: true },
|
||||
});
|
||||
if (!space?.workspacePath) {
|
||||
@ -5009,7 +5012,8 @@ async function spaceNavigationDto(
|
||||
memberships.length > 1 &&
|
||||
!spacesWithContent.has(membership.spaceId) &&
|
||||
(membership.space.deletingAt === null || membership.space.deletingAt < staleClaimBefore),
|
||||
workspace: projectWorkspaceDto(membership.space),
|
||||
// A linked folder is on the owner's host; other members see neither its path nor remote.
|
||||
workspace: actor.isDeploymentOwner ? projectWorkspaceDto(membership.space) : null,
|
||||
bots: spaceBots.map((bot) => ({
|
||||
id: bot.id,
|
||||
spaceId: bot.spaceId,
|
||||
|
||||
155
apps/desktop/e2e/browser-preview.spec.ts
Normal file
155
apps/desktop/e2e/browser-preview.spec.ts
Normal file
@ -0,0 +1,155 @@
|
||||
import http from "node:http";
|
||||
import path from "node:path";
|
||||
import { _electron as electron, expect, test } from "@playwright/test";
|
||||
import type { RakazoDesktop, RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
|
||||
|
||||
const fixture = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>Rakazo desktop browser</title></head>
|
||||
<body><main>App with a browser panel</main></body>
|
||||
</html>`;
|
||||
|
||||
type Bridge = NonNullable<RakazoDesktop["browserPreview"]>;
|
||||
|
||||
test("the in-app browser is a native view the app window lays over its panel", async () => {
|
||||
const pages = http.createServer((req, res) => {
|
||||
res.setHeader("content-type", "text/html");
|
||||
if (req.url === "/second") {
|
||||
res.end("<!doctype html><title>Second page</title><p>second</p>");
|
||||
return;
|
||||
}
|
||||
res.end(`<!doctype html><title>First page</title>
|
||||
<a id="next" href="/second">next</a>
|
||||
<a id="popup" href="/second" target="_blank">popup</a>
|
||||
<a id="file" href="file:///etc/hosts">file</a>`);
|
||||
});
|
||||
await new Promise<void>((resolve) => pages.listen(0, "127.0.0.1", resolve));
|
||||
const address = pages.address();
|
||||
if (!address || typeof address === "string") throw new Error("No fixture server");
|
||||
const origin = `http://127.0.0.1:${address.port}`;
|
||||
|
||||
const app = await electron.launch({
|
||||
args: ["."],
|
||||
cwd: path.resolve(import.meta.dirname, ".."),
|
||||
env: {
|
||||
...process.env,
|
||||
RAKAZO_WEB_URL: `data:text/html;charset=utf-8,${encodeURIComponent(fixture)}`,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const page = await app.firstWindow();
|
||||
await expect(page.getByText("App with a browser panel")).toBeVisible();
|
||||
|
||||
const call = <K extends keyof Bridge>(method: K, ...args: unknown[]) =>
|
||||
page.evaluate(
|
||||
([name, values]) => {
|
||||
const bridge = (window as typeof window & { rakazoDesktop?: RakazoDesktop }).rakazoDesktop
|
||||
?.browserPreview as unknown as Record<
|
||||
string,
|
||||
(...input: unknown[]) => Promise<unknown>
|
||||
>;
|
||||
return bridge[name as string]!(...(values as unknown[]));
|
||||
},
|
||||
[method, args] as const,
|
||||
);
|
||||
const state = () => call("state") as Promise<RakazoDesktopBrowserPreviewState>;
|
||||
const view = () =>
|
||||
app.evaluate(({ BrowserWindow }) => {
|
||||
const win = BrowserWindow.getAllWindows()[0]!;
|
||||
const child = win.contentView.children[0] as
|
||||
| (Electron.View & { webContents?: Electron.WebContents })
|
||||
| undefined;
|
||||
return child
|
||||
? {
|
||||
count: win.contentView.children.length,
|
||||
visible: child.getVisible(),
|
||||
bounds: child.getBounds(),
|
||||
url: child.webContents?.getURL(),
|
||||
partition: child.webContents?.session === win.webContents.session,
|
||||
sandbox: child.webContents?.getLastWebPreferences()?.sandbox,
|
||||
nodeIntegration: child.webContents?.getLastWebPreferences()?.nodeIntegration,
|
||||
}
|
||||
: null;
|
||||
});
|
||||
|
||||
// Pages load in a native view in their own session, never the app's.
|
||||
await call("navigate", `${origin}/`);
|
||||
await expect.poll(async () => (await state()).title).toBe("First page");
|
||||
expect(await view()).toMatchObject({
|
||||
count: 1,
|
||||
visible: false,
|
||||
url: `${origin}/`,
|
||||
partition: false,
|
||||
sandbox: true,
|
||||
nodeIntegration: false,
|
||||
});
|
||||
|
||||
await call("show", { x: 400, y: 50, width: 300, height: 200 });
|
||||
expect(await view()).toMatchObject({
|
||||
visible: true,
|
||||
bounds: { x: 400, y: 50, width: 300, height: 200 },
|
||||
});
|
||||
// Renderer bounds outside the window are clamped to it.
|
||||
await call("show", { x: -10, y: -10, width: 100_000, height: 100_000 });
|
||||
const [width, height] = await app.evaluate(({ BrowserWindow }) =>
|
||||
BrowserWindow.getAllWindows()[0]!.getContentSize(),
|
||||
);
|
||||
expect((await view())?.bounds).toEqual({ x: 0, y: 0, width, height });
|
||||
|
||||
const contents = () =>
|
||||
app.evaluate(({ BrowserWindow }, script) => {
|
||||
const win = BrowserWindow.getAllWindows()[0]!;
|
||||
const child = win.contentView.children[0] as Electron.View & {
|
||||
webContents: Electron.WebContents;
|
||||
};
|
||||
return child.webContents.executeJavaScript(script, true);
|
||||
}, "document.getElementById('next').click()");
|
||||
await contents();
|
||||
await expect.poll(async () => (await state()).title).toBe("Second page");
|
||||
expect(await state()).toMatchObject({ canGoBack: true, canGoForward: false, error: null });
|
||||
|
||||
await call("back");
|
||||
await expect.poll(async () => (await state()).title).toBe("First page");
|
||||
expect(await state()).toMatchObject({ canGoForward: true });
|
||||
await call("forward");
|
||||
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
|
||||
|
||||
// New-window links stay in the one view; file links never load.
|
||||
await call("back");
|
||||
await expect.poll(async () => (await state()).title).toBe("First page");
|
||||
await app.evaluate(({ BrowserWindow }) => {
|
||||
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
|
||||
webContents: Electron.WebContents;
|
||||
};
|
||||
return child.webContents.executeJavaScript("document.getElementById('popup').click()", true);
|
||||
});
|
||||
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
|
||||
expect(await app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().length)).toBe(1);
|
||||
await call("back");
|
||||
await expect.poll(async () => (await state()).title).toBe("First page");
|
||||
await app.evaluate(({ BrowserWindow }) => {
|
||||
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
|
||||
webContents: Electron.WebContents;
|
||||
};
|
||||
return child.webContents.executeJavaScript("document.getElementById('file').click()", true);
|
||||
});
|
||||
await page.waitForTimeout(300);
|
||||
expect((await state()).url).toBe(`${origin}/`);
|
||||
|
||||
await expect(call("navigate", "file:///etc/hosts")).rejects.toThrow(/http/);
|
||||
|
||||
// A dev server that is not running yet shows as an error, then recovers on reload.
|
||||
pages.close();
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
await call("navigate", `${origin}/gone`);
|
||||
await expect.poll(async () => (await state()).error).toMatch(/ERR_CONNECTION_REFUSED/);
|
||||
|
||||
await call("hide");
|
||||
expect((await view())?.visible).toBe(false);
|
||||
} finally {
|
||||
await app.close();
|
||||
pages.closeAllConnections();
|
||||
pages.close();
|
||||
}
|
||||
});
|
||||
@ -42,6 +42,7 @@ test("launches with a narrow preload bridge and an isolated renderer", async ()
|
||||
});
|
||||
|
||||
expect(renderer.bridgeKeys).toEqual([
|
||||
"browserPreview",
|
||||
"files",
|
||||
"localSettings",
|
||||
"oauth",
|
||||
|
||||
96
apps/desktop/src/browser-preview-policy.test.ts
Normal file
96
apps/desktop/src/browser-preview-policy.test.ts
Normal file
@ -0,0 +1,96 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
previewBounds,
|
||||
previewLoadError,
|
||||
previewShortcut,
|
||||
previewUrl,
|
||||
} from "./browser-preview-policy.js";
|
||||
|
||||
describe("previewUrl", () => {
|
||||
it("accepts only http(s) without credentials", () => {
|
||||
expect(previewUrl("http://localhost:5173")).toBe("http://localhost:5173/");
|
||||
expect(previewUrl("https://example.com/a?b=1")).toBe("https://example.com/a?b=1");
|
||||
for (const value of [
|
||||
"file:///etc/hosts",
|
||||
"javascript:alert(1)",
|
||||
"chrome://settings",
|
||||
"https://user:fake-password@example.com",
|
||||
"not a url",
|
||||
42,
|
||||
`https://example.com/${"a".repeat(9000)}`,
|
||||
]) {
|
||||
expect(previewUrl(value)).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("previewBounds", () => {
|
||||
const content = { width: 1200, height: 800 };
|
||||
|
||||
it("rounds and applies the app zoom factor", () => {
|
||||
expect(previewBounds({ x: 800.4, y: 40.6, width: 399.5, height: 700 }, content)).toEqual({
|
||||
x: 800,
|
||||
y: 41,
|
||||
width: 400,
|
||||
height: 700,
|
||||
});
|
||||
expect(previewBounds({ x: 100, y: 10, width: 200, height: 100 }, content, 1.25)).toEqual({
|
||||
x: 125,
|
||||
y: 13,
|
||||
width: 250,
|
||||
height: 125,
|
||||
});
|
||||
});
|
||||
|
||||
it("clamps to the window and rejects empty or malformed rectangles", () => {
|
||||
expect(previewBounds({ x: -50, y: -10, width: 5000, height: 5000 }, content)).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 1200,
|
||||
height: 800,
|
||||
});
|
||||
expect(previewBounds({ x: 1300, y: 0, width: 100, height: 100 }, content)).toBeNull();
|
||||
expect(previewBounds({ x: 0, y: 0, width: 0, height: 100 }, content)).toBeNull();
|
||||
expect(previewBounds({ x: "1", y: 0, width: 1, height: 1 }, content)).toBeNull();
|
||||
expect(previewBounds({ x: Number.NaN, y: 0, width: 1, height: 1 }, content)).toBeNull();
|
||||
expect(previewBounds(null, content)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("previewShortcut", () => {
|
||||
const key = (overrides: Partial<Parameters<typeof previewShortcut>[0]>) => ({
|
||||
type: "keyDown",
|
||||
key: "r",
|
||||
control: false,
|
||||
meta: false,
|
||||
alt: false,
|
||||
shift: false,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
it("maps platform browser shortcuts", () => {
|
||||
expect(previewShortcut(key({ meta: true }), "darwin")).toBe("reload");
|
||||
expect(previewShortcut(key({ meta: true, shift: true, key: "R" }), "darwin")).toBe(
|
||||
"hard-reload",
|
||||
);
|
||||
expect(previewShortcut(key({ control: true }), "linux")).toBe("reload");
|
||||
expect(previewShortcut(key({ meta: true, key: "[" }), "darwin")).toBe("back");
|
||||
expect(previewShortcut(key({ alt: true, key: "ArrowRight" }), "win32")).toBe("forward");
|
||||
expect(previewShortcut(key({ key: "F12" }), "linux")).toBe("devtools");
|
||||
expect(previewShortcut(key({ key: "F5" }), "win32")).toBe("reload");
|
||||
});
|
||||
|
||||
it("leaves typing alone", () => {
|
||||
expect(previewShortcut(key({}), "darwin")).toBeNull();
|
||||
expect(previewShortcut(key({ control: true }), "darwin")).toBeNull();
|
||||
expect(previewShortcut(key({ type: "keyUp", meta: true }), "darwin")).toBeNull();
|
||||
expect(previewShortcut(key({ key: "ArrowLeft" }), "linux")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("previewLoadError", () => {
|
||||
it("ignores superseded navigations", () => {
|
||||
expect(previewLoadError(-3, "ERR_ABORTED")).toBeNull();
|
||||
expect(previewLoadError(-102, "ERR_CONNECTION_REFUSED")).toBe("ERR_CONNECTION_REFUSED");
|
||||
});
|
||||
});
|
||||
76
apps/desktop/src/browser-preview-policy.ts
Normal file
76
apps/desktop/src/browser-preview-policy.ts
Normal file
@ -0,0 +1,76 @@
|
||||
import type { RakazoDesktopBrowserBounds } from "@rakazo/contracts";
|
||||
|
||||
/** Only plain http(s) pages load in the in-app browser; no files, schemes or URL credentials. */
|
||||
export function previewUrl(value: unknown): string | null {
|
||||
if (typeof value !== "string" || value.length > 8192) return null;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||
if (url.username || url.password) return null;
|
||||
return url.toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renderer-reported CSS-pixel bounds, converted to window DIPs and clamped to the content area
|
||||
* so a compromised renderer cannot place the view outside the panel it reserved.
|
||||
*/
|
||||
export function previewBounds(
|
||||
value: unknown,
|
||||
content: { width: number; height: number },
|
||||
zoomFactor = 1,
|
||||
): RakazoDesktopBrowserBounds | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const raw = value as Record<string, unknown>;
|
||||
const numbers = [raw.x, raw.y, raw.width, raw.height];
|
||||
if (!numbers.every((n) => typeof n === "number" && Number.isFinite(n))) return null;
|
||||
const scale = Number.isFinite(zoomFactor) && zoomFactor > 0 ? zoomFactor : 1;
|
||||
const [x, y, width, height] = (numbers as number[]).map((n) => Math.round(n * scale)) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
const left = Math.min(Math.max(0, x), content.width);
|
||||
const top = Math.min(Math.max(0, y), content.height);
|
||||
const right = Math.min(Math.max(left, x + width), content.width);
|
||||
const bottom = Math.min(Math.max(top, y + height), content.height);
|
||||
if (right - left < 1 || bottom - top < 1) return null;
|
||||
return { x: left, y: top, width: right - left, height: bottom - top };
|
||||
}
|
||||
|
||||
export type PreviewShortcut = "reload" | "hard-reload" | "back" | "forward" | "devtools";
|
||||
|
||||
/** Browser keyboard shortcuts while focus is inside the page. */
|
||||
export function previewShortcut(
|
||||
input: {
|
||||
type: string;
|
||||
key: string;
|
||||
control: boolean;
|
||||
meta: boolean;
|
||||
alt: boolean;
|
||||
shift: boolean;
|
||||
},
|
||||
platform: string,
|
||||
): PreviewShortcut | null {
|
||||
if (input.type !== "keyDown") return null;
|
||||
const mod = platform === "darwin" ? input.meta : input.control;
|
||||
const key = input.key.length === 1 ? input.key.toLowerCase() : input.key;
|
||||
if (key === "F5") return input.shift ? "hard-reload" : "reload";
|
||||
if (key === "F12") return "devtools";
|
||||
if (mod && key === "r") return input.shift ? "hard-reload" : "reload";
|
||||
if (mod && input.alt && key === "i") return "devtools";
|
||||
if (platform === "darwin" ? input.meta && !input.alt : input.alt && !input.control) {
|
||||
if (key === (platform === "darwin" ? "[" : "ArrowLeft")) return "back";
|
||||
if (key === (platform === "darwin" ? "]" : "ArrowRight")) return "forward";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Load failures worth showing; an aborted load (-3) is a superseded navigation, not an error. */
|
||||
export function previewLoadError(errorCode: number, description: string): string | null {
|
||||
if (errorCode === -3 || errorCode === 0) return null;
|
||||
return description || `Load failed (${errorCode})`;
|
||||
}
|
||||
189
apps/desktop/src/browser-preview.ts
Normal file
189
apps/desktop/src/browser-preview.ts
Normal file
@ -0,0 +1,189 @@
|
||||
import type { RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
|
||||
import { type BrowserWindow, WebContentsView } from "electron";
|
||||
import {
|
||||
previewBounds,
|
||||
previewLoadError,
|
||||
previewShortcut,
|
||||
previewUrl,
|
||||
} from "./browser-preview-policy.js";
|
||||
|
||||
/** Pages share one persistent profile, separate from the app's own session and cookies. */
|
||||
const PREVIEW_PARTITION = "persist:rakazo-browser";
|
||||
export const PREVIEW_STATE_CHANNEL = "desktop.browserPreview.state";
|
||||
|
||||
/**
|
||||
* The in-app browser's native page: one WebContentsView laid over the panel the app window
|
||||
* reserves. It belongs to one app window at a time and is torn down with it.
|
||||
*/
|
||||
export class BrowserPreviewController {
|
||||
private host: BrowserWindow | null = null;
|
||||
private view: WebContentsView | null = null;
|
||||
private error: string | null = null;
|
||||
|
||||
state(win: BrowserWindow): RakazoDesktopBrowserPreviewState {
|
||||
const contents = this.host === win ? this.view?.webContents : undefined;
|
||||
if (!contents || contents.isDestroyed()) {
|
||||
return {
|
||||
url: "",
|
||||
title: "",
|
||||
loading: false,
|
||||
canGoBack: false,
|
||||
canGoForward: false,
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
const url = contents.getURL();
|
||||
return {
|
||||
url,
|
||||
title: contents.getTitle() === url ? "" : contents.getTitle(),
|
||||
loading: contents.isLoading(),
|
||||
canGoBack: contents.navigationHistory.canGoBack(),
|
||||
canGoForward: contents.navigationHistory.canGoForward(),
|
||||
error: this.error,
|
||||
};
|
||||
}
|
||||
|
||||
show(win: BrowserWindow, rawBounds: unknown) {
|
||||
const view = this.viewFor(win);
|
||||
const [width, height] = win.getContentSize();
|
||||
const bounds = previewBounds(
|
||||
rawBounds,
|
||||
{ width: width ?? 0, height: height ?? 0 },
|
||||
win.webContents.getZoomFactor(),
|
||||
);
|
||||
if (!bounds) {
|
||||
view.setVisible(false);
|
||||
return;
|
||||
}
|
||||
view.setBounds(bounds);
|
||||
view.setVisible(true);
|
||||
}
|
||||
|
||||
hide(win: BrowserWindow) {
|
||||
if (this.host === win) this.view?.setVisible(false);
|
||||
}
|
||||
|
||||
async navigate(win: BrowserWindow, rawUrl: unknown) {
|
||||
const url = previewUrl(rawUrl);
|
||||
if (!url) throw new Error("Only http(s) pages can open here.");
|
||||
this.error = null;
|
||||
// Failures surface through did-fail-load as page state, not as a rejected call.
|
||||
await this.viewFor(win)
|
||||
.webContents.loadURL(url)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
back(win: BrowserWindow) {
|
||||
const history = this.contentsFor(win)?.navigationHistory;
|
||||
if (history?.canGoBack()) history.goBack();
|
||||
}
|
||||
|
||||
forward(win: BrowserWindow) {
|
||||
const history = this.contentsFor(win)?.navigationHistory;
|
||||
if (history?.canGoForward()) history.goForward();
|
||||
}
|
||||
|
||||
reload(win: BrowserWindow, ignoreCache = false) {
|
||||
const contents = this.contentsFor(win);
|
||||
if (!contents?.getURL()) return;
|
||||
this.error = null;
|
||||
if (ignoreCache) contents.reloadIgnoringCache();
|
||||
else contents.reload();
|
||||
}
|
||||
|
||||
stop(win: BrowserWindow) {
|
||||
this.contentsFor(win)?.stop();
|
||||
}
|
||||
|
||||
openDevTools(win: BrowserWindow) {
|
||||
const contents = this.contentsFor(win);
|
||||
if (contents?.getURL()) contents.openDevTools({ mode: "detach" });
|
||||
}
|
||||
|
||||
private contentsFor(win: BrowserWindow) {
|
||||
const contents = this.host === win ? this.view?.webContents : undefined;
|
||||
return contents && !contents.isDestroyed() ? contents : undefined;
|
||||
}
|
||||
|
||||
private viewFor(win: BrowserWindow): WebContentsView {
|
||||
if (this.host === win && this.view && !this.view.webContents.isDestroyed()) return this.view;
|
||||
this.destroy();
|
||||
const view = new WebContentsView({
|
||||
webPreferences: {
|
||||
partition: PREVIEW_PARTITION,
|
||||
sandbox: true,
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
webSecurity: true,
|
||||
spellcheck: false,
|
||||
},
|
||||
});
|
||||
view.setVisible(false);
|
||||
const contents = view.webContents;
|
||||
const publish = () => {
|
||||
if (!win.isDestroyed()) win.webContents.send(PREVIEW_STATE_CHANNEL, this.state(win));
|
||||
};
|
||||
// One tab: links that want a new window open here instead.
|
||||
contents.setWindowOpenHandler(({ url }) => {
|
||||
const target = previewUrl(url);
|
||||
if (target) void contents.loadURL(target).catch(() => undefined);
|
||||
return { action: "deny" };
|
||||
});
|
||||
contents.on("will-navigate", (event, url) => {
|
||||
if (!previewUrl(url)) event.preventDefault();
|
||||
});
|
||||
contents.on("will-redirect", (event, url) => {
|
||||
if (!previewUrl(url)) event.preventDefault();
|
||||
});
|
||||
contents.on("did-start-loading", publish);
|
||||
contents.on("did-stop-loading", publish);
|
||||
contents.on("did-navigate", () => {
|
||||
this.error = null;
|
||||
publish();
|
||||
});
|
||||
contents.on("did-navigate-in-page", publish);
|
||||
contents.on("page-title-updated", publish);
|
||||
contents.on("did-fail-load", (_event, code, description, _url, isMainFrame) => {
|
||||
if (!isMainFrame) return;
|
||||
const error = previewLoadError(code, description);
|
||||
if (error === null) return;
|
||||
this.error = error;
|
||||
publish();
|
||||
});
|
||||
contents.on("render-process-gone", (_event, details) => {
|
||||
this.error = details.reason === "clean-exit" ? null : "The page stopped responding.";
|
||||
publish();
|
||||
});
|
||||
contents.on("before-input-event", (event, input) => {
|
||||
const shortcut = previewShortcut(input, process.platform);
|
||||
if (!shortcut) return;
|
||||
event.preventDefault();
|
||||
if (shortcut === "reload" || shortcut === "hard-reload") {
|
||||
this.reload(win, shortcut === "hard-reload");
|
||||
} else if (shortcut === "back") {
|
||||
this.back(win);
|
||||
} else if (shortcut === "forward") {
|
||||
this.forward(win);
|
||||
} else {
|
||||
this.openDevTools(win);
|
||||
}
|
||||
});
|
||||
win.contentView.addChildView(view);
|
||||
win.once("closed", () => {
|
||||
if (this.host === win) this.destroy();
|
||||
});
|
||||
this.host = win;
|
||||
this.view = view;
|
||||
return view;
|
||||
}
|
||||
|
||||
private destroy() {
|
||||
const { host, view } = this;
|
||||
this.host = null;
|
||||
this.view = null;
|
||||
this.error = null;
|
||||
if (!view) return;
|
||||
if (host && !host.isDestroyed()) host.contentView.removeChildView(view);
|
||||
if (!view.webContents.isDestroyed()) view.webContents.close();
|
||||
}
|
||||
}
|
||||
@ -21,6 +21,7 @@ import {
|
||||
LAUNCH_CHECK_DELAY_MS,
|
||||
} from "./auto-update.js";
|
||||
import { openBrowserAuth } from "./browser-auth.js";
|
||||
import { BrowserPreviewController } from "./browser-preview.js";
|
||||
import { DOCKER_INSTALL_LINKS, isDesktopSetupLink, runDocker } from "./docker-cli.js";
|
||||
import { requestLocalSettings } from "./local-settings.js";
|
||||
import {
|
||||
@ -1130,6 +1131,35 @@ app.whenReady().then(async () => {
|
||||
},
|
||||
);
|
||||
ipcMain.handle("desktop.platform", () => process.platform);
|
||||
const browserPreview = new BrowserPreviewController();
|
||||
// The in-app browser answers only the app window's own top frame, never a page inside it.
|
||||
const previewHost = (event: Electron.IpcMainInvokeEvent) => {
|
||||
if (!fromMainWindow(event) || event.senderFrame !== event.sender.mainFrame) {
|
||||
throw new Error("The in-app browser is not available here.");
|
||||
}
|
||||
return mainWindow!;
|
||||
};
|
||||
ipcMain.handle("desktop.browserPreview.state", (event) =>
|
||||
browserPreview.state(previewHost(event)),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.show", (event, bounds: unknown) =>
|
||||
browserPreview.show(previewHost(event), bounds),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.hide", (event) => browserPreview.hide(previewHost(event)));
|
||||
ipcMain.handle("desktop.browserPreview.navigate", (event, url: unknown) =>
|
||||
browserPreview.navigate(previewHost(event), url),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.back", (event) => browserPreview.back(previewHost(event)));
|
||||
ipcMain.handle("desktop.browserPreview.forward", (event) =>
|
||||
browserPreview.forward(previewHost(event)),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.reload", (event) =>
|
||||
browserPreview.reload(previewHost(event)),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.stop", (event) => browserPreview.stop(previewHost(event)));
|
||||
ipcMain.handle("desktop.browserPreview.openDevTools", (event) =>
|
||||
browserPreview.openDevTools(previewHost(event)),
|
||||
);
|
||||
ipcMain.handle("desktop.files.pickDirectory", async (event) => {
|
||||
if (!fromMainWindow(event)) return null;
|
||||
const result = await dialog.showOpenDialog(windowFrom(event)!, {
|
||||
|
||||
@ -21,6 +21,22 @@ contextBridge.exposeInMainWorld("rakazoDesktop", {
|
||||
download: () => ipcRenderer.invoke("desktop.update.download"),
|
||||
install: () => ipcRenderer.invoke("desktop.update.install"),
|
||||
},
|
||||
browserPreview: {
|
||||
state: () => ipcRenderer.invoke("desktop.browserPreview.state"),
|
||||
show: (bounds) => ipcRenderer.invoke("desktop.browserPreview.show", bounds),
|
||||
hide: () => ipcRenderer.invoke("desktop.browserPreview.hide"),
|
||||
navigate: (url) => ipcRenderer.invoke("desktop.browserPreview.navigate", url),
|
||||
back: () => ipcRenderer.invoke("desktop.browserPreview.back"),
|
||||
forward: () => ipcRenderer.invoke("desktop.browserPreview.forward"),
|
||||
reload: () => ipcRenderer.invoke("desktop.browserPreview.reload"),
|
||||
stop: () => ipcRenderer.invoke("desktop.browserPreview.stop"),
|
||||
openDevTools: () => ipcRenderer.invoke("desktop.browserPreview.openDevTools"),
|
||||
onState: (listener) => {
|
||||
const handler = (_event, state) => listener(state);
|
||||
ipcRenderer.on("desktop.browserPreview.state", handler);
|
||||
return () => ipcRenderer.off("desktop.browserPreview.state", handler);
|
||||
},
|
||||
},
|
||||
oauth: {
|
||||
open: (url) => ipcRenderer.invoke("desktop.oauth.open", url),
|
||||
cancel: (url) => ipcRenderer.invoke("desktop.oauth.cancel", url),
|
||||
|
||||
@ -32,6 +32,7 @@ describe("desktop preload bridge", () => {
|
||||
expect(globalName).toBe("rakazoDesktop");
|
||||
expect(bridge.platform).toBe("linux");
|
||||
expect(Object.keys(bridge).sort()).toEqual([
|
||||
"browserPreview",
|
||||
"files",
|
||||
"localSettings",
|
||||
"oauth",
|
||||
@ -46,6 +47,18 @@ describe("desktop preload bridge", () => {
|
||||
"toggleMaximize",
|
||||
]);
|
||||
expect(Object.keys(bridge.update).sort()).toEqual(["check", "download", "install", "state"]);
|
||||
expect(Object.keys(bridge.browserPreview ?? {}).sort()).toEqual([
|
||||
"back",
|
||||
"forward",
|
||||
"hide",
|
||||
"navigate",
|
||||
"onState",
|
||||
"openDevTools",
|
||||
"reload",
|
||||
"show",
|
||||
"state",
|
||||
"stop",
|
||||
]);
|
||||
|
||||
await bridge.files?.pickDirectory();
|
||||
await bridge.oauth.open?.("https://provider.example.com/authorize");
|
||||
@ -58,6 +71,8 @@ describe("desktop preload bridge", () => {
|
||||
await bridge.update.check();
|
||||
await bridge.update.download();
|
||||
await bridge.update.install();
|
||||
await bridge.browserPreview?.navigate("http://localhost:5173/");
|
||||
await bridge.browserPreview?.show({ x: 1, y: 2, width: 3, height: 4 });
|
||||
expect(invoke.mock.calls.map(([channel]) => channel)).toEqual([
|
||||
"desktop.files.pickDirectory",
|
||||
"desktop.oauth.open",
|
||||
@ -70,6 +85,8 @@ describe("desktop preload bridge", () => {
|
||||
"desktop.update.check",
|
||||
"desktop.update.download",
|
||||
"desktop.update.install",
|
||||
"desktop.browserPreview.navigate",
|
||||
"desktop.browserPreview.show",
|
||||
]);
|
||||
});
|
||||
|
||||
@ -77,6 +94,7 @@ describe("desktop preload bridge", () => {
|
||||
const { exposeInMainWorld } = runPreload("preload.cjs");
|
||||
const [, bridge] = exposeInMainWorld.mock.calls[0] as [string, Record<string, unknown>];
|
||||
expect(Object.keys(bridge).sort()).toEqual([
|
||||
"browserPreview",
|
||||
"files",
|
||||
"localSettings",
|
||||
"oauth",
|
||||
|
||||
@ -17,10 +17,12 @@ import {
|
||||
buildComposerMentionOptions,
|
||||
type ComposerMention,
|
||||
cloudAgentHttpsUrl,
|
||||
deriveWorkspaceFocus,
|
||||
isApprovalAskBlock,
|
||||
isRunTerminalEvent,
|
||||
isSecretAskBlock,
|
||||
latestAnswerableAskMessageId,
|
||||
liveScreenRunId,
|
||||
mentionChipKey,
|
||||
projectMessageReactions,
|
||||
resolveComposerSendPlan,
|
||||
@ -288,6 +290,8 @@ function Thread() {
|
||||
? { botId }
|
||||
: undefined;
|
||||
const [snap, setSnap] = useState<MobileSnapshot | null>(null);
|
||||
/** Run that last pointed at the bot's screen, so the live screen is offered only during real screen work. */
|
||||
const [screenRunId, setScreenRunId] = useState<string | null>(null);
|
||||
const activeThreadId = useRef<string | undefined>(undefined);
|
||||
const [draft, setDraft] = useState("");
|
||||
const [mentionQuery, setMentionQuery] = useState<string | null>(null);
|
||||
@ -860,6 +864,7 @@ function Thread() {
|
||||
}
|
||||
expandedHistoryThread.current = null;
|
||||
historyEpoch.current += 1;
|
||||
setScreenRunId(null);
|
||||
const abort = new AbortController();
|
||||
void (async () => {
|
||||
// Pending search jumps load the around-page separately; avoid replacing it with latest.
|
||||
@ -885,6 +890,8 @@ function Thread() {
|
||||
(event) => {
|
||||
cursor = Math.max(cursor, event.seq ?? -1);
|
||||
retryMs = 250;
|
||||
const screenRun = groupId ? null : liveScreenRunId(event);
|
||||
if (screenRun) setScreenRunId(screenRun);
|
||||
if (
|
||||
event.type === "thread.progress" ||
|
||||
event.type === "agent.tool.called" ||
|
||||
@ -1271,6 +1278,23 @@ function Thread() {
|
||||
}
|
||||
|
||||
const answerableAskMessageId = latestAnswerableAskMessageId(snap);
|
||||
const screenFocus = useMemo(() => {
|
||||
const computer = snap?.computer;
|
||||
if (!snap || !computer || groupId) return null;
|
||||
const focus = deriveWorkspaceFocus(
|
||||
{
|
||||
...snap,
|
||||
computer: {
|
||||
kind: computer.kind ?? "",
|
||||
state: computer.state,
|
||||
screenAvailable: computer.screenAvailable,
|
||||
takeoverRequested: computer.takeoverRequested ?? false,
|
||||
},
|
||||
},
|
||||
{ screenRunId },
|
||||
);
|
||||
return focus.pane === "screen" ? focus : null;
|
||||
}, [groupId, screenRunId, snap]);
|
||||
const runError = snap?.run?.status === "failed" ? (snap.run.error ?? null) : null;
|
||||
const liveMessages = useMemo(() => [...visibleMessages].reverse(), [visibleMessages]);
|
||||
const messagesById = useMemo(
|
||||
@ -1668,6 +1692,43 @@ function Thread() {
|
||||
) : null}
|
||||
</View>
|
||||
<View style={{ paddingBottom: keyboardVisible ? 12 : Math.max(insets.bottom + 12, 24) }}>
|
||||
{screenFocus && botId ? (
|
||||
<Pressable
|
||||
accessibilityRole="button"
|
||||
testID="live-screen-pill"
|
||||
onPress={() =>
|
||||
router.push({
|
||||
pathname: "/computer",
|
||||
params: { botId, name: displayName ?? t("Bot") },
|
||||
})
|
||||
}
|
||||
style={({ pressed }) => ({
|
||||
alignSelf: "center",
|
||||
marginTop: 10,
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
borderRadius: 999,
|
||||
borderWidth: 1,
|
||||
borderColor: native.fillPressed,
|
||||
backgroundColor: pressed ? native.fillPressed : native.fill,
|
||||
paddingHorizontal: 14,
|
||||
paddingVertical: 8,
|
||||
})}
|
||||
>
|
||||
<View
|
||||
style={{
|
||||
width: 8,
|
||||
height: 8,
|
||||
borderRadius: 4,
|
||||
backgroundColor: screenFocus.takeover ? tokens.warning : tokens.success,
|
||||
}}
|
||||
/>
|
||||
<Text style={{ color: tokens.foreground, fontSize: 14, fontWeight: "500" }}>
|
||||
{screenFocus.takeover ? t("Needs takeover") : t("Open computer")}
|
||||
</Text>
|
||||
</Pressable>
|
||||
) : null}
|
||||
{replyTarget ? (
|
||||
<View
|
||||
style={{
|
||||
|
||||
@ -790,9 +790,11 @@ export type MobileSnapshot = {
|
||||
activeRuns?: Array<{ id: string; botId?: string; status: string }>;
|
||||
members?: MobileGroup["members"];
|
||||
computer?: {
|
||||
kind?: string;
|
||||
state: string;
|
||||
controlHolder: string;
|
||||
screenAvailable: boolean;
|
||||
takeoverRequested?: boolean;
|
||||
mode: ComputerMode;
|
||||
busyBotName: string | null;
|
||||
};
|
||||
@ -896,6 +898,7 @@ type ThreadEvent = {
|
||||
type: string;
|
||||
seq?: number;
|
||||
runId?: string;
|
||||
createdAt?: string;
|
||||
payload?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
@ -1074,7 +1077,7 @@ export function applyMobileThreadEvent(
|
||||
messages: [...remaining, streaming],
|
||||
};
|
||||
}
|
||||
if (event.type === "agent.tool.completed") {
|
||||
if (event.type === "agent.tool.completed" || event.type === "thread.computer") {
|
||||
return { ...prev, cursor: event.seq ?? prev.cursor };
|
||||
}
|
||||
if (event.type === "thread.subagent") {
|
||||
|
||||
104
apps/web/e2e/adaptive-workspace.spec.ts
Normal file
104
apps/web/e2e/adaptive-workspace.spec.ts
Normal file
@ -0,0 +1,104 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers";
|
||||
|
||||
const SCREEN_URL = "https://screen.example/embed.html";
|
||||
|
||||
test("the bot's screen opens beside the chat while it works and steps back after", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
await signup(page, `workspace-screen-${Date.now()}@rakazo.test`, "password12", "Screen Pane");
|
||||
await completeOnboarding(page);
|
||||
const botId = activeBotId(page);
|
||||
await rpc(page, "computer/boot", { botId });
|
||||
await page.route("https://screen.example/**", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "text/html",
|
||||
body: `<!doctype html><title>Desktop</title><body style="margin:0;background:#1d1d20;display:grid;place-items:center;height:100vh">
|
||||
<input style="width:40%;padding:8px;font:16px system-ui" placeholder="Search">`,
|
||||
}),
|
||||
);
|
||||
await page.route("**/rpc/computer/screenUrl", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ json: { url: SCREEN_URL } }),
|
||||
}),
|
||||
);
|
||||
|
||||
const sidePanel = page.getByTestId("side-panel");
|
||||
await expect(sidePanel).toHaveAttribute("data-panel", "closed");
|
||||
|
||||
await rpc(page, "threads/send", { botId, text: "use your screen and type workspace-demo" });
|
||||
|
||||
await expect(sidePanel).toHaveAttribute("data-panel", "computer");
|
||||
await expect(sidePanel).toHaveAttribute("data-auto", "");
|
||||
await expect(page.getByTestId("computer-preview").locator("iframe")).toHaveAttribute(
|
||||
"src",
|
||||
SCREEN_URL,
|
||||
);
|
||||
// The live screen gets more room than a settings panel.
|
||||
expect((await sidePanel.boundingBox())!.width).toBeGreaterThan(400);
|
||||
await captureScreenshot(page, testInfo, "workspace-screen-beside-chat");
|
||||
|
||||
// Once the work is done the conversation gets its full width back.
|
||||
await expect(sidePanel).toHaveAttribute("data-panel", "closed", { timeout: 20_000 });
|
||||
});
|
||||
|
||||
test("a report the bot presents opens beside the conversation", async ({ page }, testInfo) => {
|
||||
await signup(page, `workspace-report-${Date.now()}@rakazo.test`, "password12", "Report Pane");
|
||||
await completeOnboarding(page);
|
||||
|
||||
const composer = page.getByPlaceholder(/Message/);
|
||||
await composer.fill("present the report");
|
||||
await page.keyboard.press("Enter");
|
||||
|
||||
const pane = page.getByTestId("artifact-pane");
|
||||
await expect(pane).toBeVisible({ timeout: 30_000 });
|
||||
await expect(pane.getByRole("heading", { name: "report.md" })).toBeVisible();
|
||||
await expect(pane.getByRole("heading", { name: "Report", exact: true })).toBeVisible();
|
||||
await expect(pane.getByRole("cell", { name: "Research" })).toBeVisible();
|
||||
await expect(composer).toBeVisible();
|
||||
await captureScreenshot(page, testInfo, "workspace-report-beside-chat");
|
||||
|
||||
const downloadPromise = page.waitForEvent("download");
|
||||
await pane.getByRole("button", { name: "Download report.md" }).click();
|
||||
expect((await downloadPromise).suggestedFilename()).toBe("report.md");
|
||||
|
||||
await pane.getByRole("button", { name: "Close panel" }).click();
|
||||
await expect(pane).toHaveCount(0);
|
||||
|
||||
// The file card in the chat opens the same pane again.
|
||||
await page.getByRole("button", { name: "Preview report.md" }).click();
|
||||
await expect(pane.getByRole("heading", { name: "Report", exact: true })).toBeVisible();
|
||||
});
|
||||
|
||||
test("a question that scrolled away is one click back", async ({ page }, testInfo) => {
|
||||
await page.setViewportSize({ width: 1280, height: 520 });
|
||||
await signup(page, `workspace-ask-${Date.now()}@rakazo.test`, "password12", "Ask Jump");
|
||||
await completeOnboarding(page);
|
||||
|
||||
const composer = page.getByPlaceholder(/Message/);
|
||||
// Earlier work gives the thread enough history to scroll back through.
|
||||
for (const task of ["sort the inbox", "draft the agenda", "book the room"]) {
|
||||
await composer.fill(task);
|
||||
await page.keyboard.press("Enter");
|
||||
await expect(page.getByText(`done. i handled: ${task}`, { exact: true })).toBeVisible({
|
||||
timeout: 30_000,
|
||||
});
|
||||
}
|
||||
await composer.fill("ask me which city to use");
|
||||
await page.keyboard.press("Enter");
|
||||
const sendIt = page.getByRole("button", { name: "Send it" });
|
||||
await expect(sendIt).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
const jump = page.getByTestId("needs-you-jump");
|
||||
await expect(jump).toHaveCount(0);
|
||||
await page.getByTestId("transcript").hover();
|
||||
await page.mouse.wheel(0, -20_000);
|
||||
await expect(jump).toBeVisible();
|
||||
await captureScreenshot(page, testInfo, "workspace-needs-you-jump");
|
||||
|
||||
await jump.click();
|
||||
await expect(sendIt).toBeInViewport();
|
||||
await expect(sendIt).toBeFocused();
|
||||
await expect(jump).toHaveCount(0);
|
||||
});
|
||||
53
apps/web/e2e/agent-cursor.spec.ts
Normal file
53
apps/web/e2e/agent-cursor.spec.ts
Normal file
@ -0,0 +1,53 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers";
|
||||
|
||||
const SCREEN_URL = "https://screen.example/embed.html";
|
||||
|
||||
test("the bot's cursor glides to where it clicks on its live screen", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
await signup(page, `agent-cursor-${Date.now()}@rakazo.test`, "password12", "Cursor Watcher");
|
||||
await completeOnboarding(page);
|
||||
const botId = activeBotId(page);
|
||||
await rpc(page, "computer/boot", { botId });
|
||||
// Stand in for the noVNC viewer: a 16:10 desktop with a field in the middle.
|
||||
await page.route("https://screen.example/**", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "text/html",
|
||||
body: `<!doctype html><title>Desktop</title><body style="margin:0;background:#1d1d20;display:grid;place-items:center;height:100vh">
|
||||
<input style="width:40%;padding:8px;font:16px system-ui" placeholder="Search">`,
|
||||
}),
|
||||
);
|
||||
await page.route("**/rpc/computer/screenUrl", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ json: { url: SCREEN_URL } }),
|
||||
}),
|
||||
);
|
||||
|
||||
await page.getByTitle("Agent computer").click();
|
||||
const preview = page.getByTestId("computer-preview");
|
||||
await expect(preview.locator("iframe")).toHaveAttribute("src", SCREEN_URL);
|
||||
const cursor = preview.getByTestId("agent-cursor");
|
||||
await expect(cursor).toHaveCSS("opacity", "0");
|
||||
|
||||
// The scripted runtime clicks the screen center (640, 400) and types.
|
||||
await rpc(page, "threads/send", { botId, text: "use your screen and type cursor-demo" });
|
||||
|
||||
await expect(cursor).toHaveCSS("opacity", "1");
|
||||
await expect(cursor).toHaveAttribute("data-action", /click|type/);
|
||||
const layer = (await preview.getByTestId("agent-cursor-layer").boundingBox())!;
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const tip = (await cursor.boundingBox())!;
|
||||
return [
|
||||
Math.round(tip.x - layer.x - layer.width / 2),
|
||||
Math.round(tip.y - layer.y - layer.height / 2),
|
||||
];
|
||||
})
|
||||
.toEqual([0, 0]);
|
||||
await captureScreenshot(page, testInfo, "agent-cursor-on-screen");
|
||||
|
||||
// It fades once the bot stops pointing, so it never lingers over the screen.
|
||||
await expect(cursor).toHaveCSS("opacity", "0", { timeout: 10_000 });
|
||||
});
|
||||
@ -27,6 +27,8 @@ test("agent-attached files appear as downloadable cards", async ({ page }, testI
|
||||
test("agent-attached Markdown opens a rendered preview and can be downloaded", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
// Narrow windows preview in a dialog; wide ones open the file beside the chat.
|
||||
await page.setViewportSize({ width: 900, height: 800 });
|
||||
const stamp = Date.now();
|
||||
await signup(page, `markdown-preview-${stamp}@rakazo.test`, "password12", "Markdown Preview");
|
||||
await completeOnboarding(page);
|
||||
|
||||
@ -69,9 +69,10 @@ test("takeover, routine, plugins, and export are reachable", async ({ page }, te
|
||||
})
|
||||
.toBe("waiting_takeover");
|
||||
await captureScreenshot(page, testInfo, "08-protected-input-request");
|
||||
await page.getByTitle("Agent computer").click();
|
||||
// A takeover request brings the bot's screen beside the chat on its own.
|
||||
const sidePanel = page.getByTestId("side-panel");
|
||||
await expect(sidePanel).toHaveCSS("width", "384px");
|
||||
await expect(sidePanel).toHaveAttribute("data-panel", "computer");
|
||||
await expect(sidePanel).toHaveAttribute("data-auto", "");
|
||||
const [mainBox, panelBox] = await Promise.all([
|
||||
page.locator("main").boundingBox(),
|
||||
sidePanel.boundingBox(),
|
||||
|
||||
@ -303,9 +303,11 @@ test("create group from + and see two bots in one transcript", async ({ page },
|
||||
const groupMarkdown = page.getByRole("button", { name: "Preview group-preview.md" });
|
||||
await expect(groupMarkdown).toBeVisible({ timeout: 30_000 });
|
||||
await groupMarkdown.click();
|
||||
const markdownDialog = page.getByRole("dialog", { name: "group-preview.md" });
|
||||
await expect(markdownDialog.getByRole("heading", { name: "Group artifact" })).toBeVisible();
|
||||
await markdownDialog.getByRole("button", { name: "Close preview" }).click();
|
||||
// Wide windows open the file beside the group conversation.
|
||||
const markdownPane = page.getByTestId("artifact-pane");
|
||||
await expect(markdownPane.getByRole("heading", { name: "Group artifact" })).toBeVisible();
|
||||
await markdownPane.getByRole("button", { name: "Close panel" }).click();
|
||||
await expect(markdownPane).toHaveCount(0);
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await expect(page.getByRole("button", { name: "Open navigation" })).toBeVisible();
|
||||
|
||||
159
apps/web/e2e/in-app-browser.spec.ts
Normal file
159
apps/web/e2e/in-app-browser.spec.ts
Normal file
@ -0,0 +1,159 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { captureScreenshot, completeOnboarding, signup } from "./helpers";
|
||||
|
||||
test("the in-app browser previews a page and follows it back to the browser", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
await signup(page, `in-app-browser-${Date.now()}@rakazo.test`, "password12", "Previewer");
|
||||
await completeOnboarding(page);
|
||||
// Context-wide so the "Open in browser" tab is served too.
|
||||
await page.context().route("https://preview.example/**", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "text/html",
|
||||
body: `<!doctype html><title>Preview</title><body style="font:16px system-ui;padding:24px">
|
||||
<h1>Live preview</h1><p>Rendered inside the app.</p>`,
|
||||
}),
|
||||
);
|
||||
|
||||
await page.getByRole("button", { name: "Browser" }).click();
|
||||
const browser = page.getByTestId("in-app-browser");
|
||||
const address = browser.getByRole("textbox", { name: "Address" });
|
||||
await expect(address).toBeFocused();
|
||||
|
||||
await address.fill("not an address");
|
||||
await address.press("Enter");
|
||||
await expect(address).toHaveAttribute("aria-invalid", "true");
|
||||
|
||||
await address.fill("preview.example/app");
|
||||
await address.press("Enter");
|
||||
await expect(address).toHaveValue("https://preview.example/app");
|
||||
const frame = browser.getByTestId("in-app-browser-frame");
|
||||
await expect(frame).toHaveAttribute("src", "https://preview.example/app");
|
||||
await expect(
|
||||
page.frameLocator('[data-testid="in-app-browser-frame"]').getByRole("heading"),
|
||||
).toHaveText("Live preview");
|
||||
await captureScreenshot(page, testInfo, "in-app-browser");
|
||||
|
||||
const popup = page.waitForEvent("popup");
|
||||
await browser.getByRole("button", { name: "Open in browser" }).click();
|
||||
expect((await popup).url()).toBe("https://preview.example/app");
|
||||
|
||||
await browser.getByRole("button", { name: "Close panel" }).click();
|
||||
await expect(browser).toHaveCount(0);
|
||||
// The last address comes back when the browser reopens.
|
||||
await page.getByRole("button", { name: "Browser" }).click();
|
||||
await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute(
|
||||
"src",
|
||||
"https://preview.example/app",
|
||||
);
|
||||
});
|
||||
|
||||
test("on desktop the page is a native view glued to the panel", async ({ page }) => {
|
||||
await page.addInitScript(() => {
|
||||
const calls: Array<[string, unknown?]> = [];
|
||||
let listener: ((state: unknown) => void) | undefined;
|
||||
let state = {
|
||||
url: "",
|
||||
title: "",
|
||||
loading: false,
|
||||
canGoBack: false,
|
||||
canGoForward: false,
|
||||
error: null as string | null,
|
||||
};
|
||||
const record = (name: string) => async (value?: unknown) => {
|
||||
calls.push([name, value]);
|
||||
};
|
||||
Object.assign(window, { __browserCalls: calls });
|
||||
Object.defineProperty(window, "rakazoDesktop", {
|
||||
value: {
|
||||
platform: "linux",
|
||||
window: {
|
||||
close: async () => {},
|
||||
minimize: async () => {},
|
||||
toggleMaximize: async () => {},
|
||||
state: async () => ({ minimized: false, maximized: false, fullScreen: false }),
|
||||
},
|
||||
oauth: { onCallback: () => () => {} },
|
||||
update: {
|
||||
state: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
||||
check: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
||||
download: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
||||
install: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
||||
},
|
||||
browserPreview: {
|
||||
state: async () => state,
|
||||
onState: (next: (value: unknown) => void) => {
|
||||
listener = next;
|
||||
return () => {
|
||||
listener = undefined;
|
||||
};
|
||||
},
|
||||
navigate: async (url: string) => {
|
||||
calls.push(["navigate", url]);
|
||||
state = { ...state, url, title: "Dev server", loading: false, canGoBack: true };
|
||||
listener?.(state);
|
||||
},
|
||||
show: record("show"),
|
||||
hide: record("hide"),
|
||||
back: record("back"),
|
||||
forward: record("forward"),
|
||||
reload: record("reload"),
|
||||
stop: record("stop"),
|
||||
openDevTools: record("openDevTools"),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
await signup(page, `native-browser-${Date.now()}@rakazo.test`, "password12", "Desktop");
|
||||
await completeOnboarding(page);
|
||||
const calls = () =>
|
||||
page.evaluate(
|
||||
() => (window as unknown as { __browserCalls: Array<[string, unknown?]> }).__browserCalls,
|
||||
);
|
||||
|
||||
await page.getByRole("button", { name: "Browser" }).click();
|
||||
const browser = page.getByTestId("in-app-browser");
|
||||
await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173");
|
||||
await browser.getByRole("textbox", { name: "Address" }).press("Enter");
|
||||
await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]);
|
||||
await expect(browser.getByRole("button", { name: "Back" })).toBeEnabled();
|
||||
await expect(browser.getByRole("button", { name: "Forward" })).toBeDisabled();
|
||||
await expect(browser.getByTestId("in-app-browser-frame")).toHaveCount(0);
|
||||
|
||||
// The reserved page area and the native view's bounds stay identical.
|
||||
const viewport = browser.getByTestId("in-app-browser-viewport");
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const box = (await viewport.boundingBox())!;
|
||||
const shown = (await calls()).filter(([name]) => name === "show").at(-1)?.[1];
|
||||
return (
|
||||
JSON.stringify(shown) ===
|
||||
JSON.stringify({
|
||||
x: Math.round(box.x),
|
||||
y: Math.round(box.y),
|
||||
width: Math.round(box.width),
|
||||
height: Math.round(box.height),
|
||||
})
|
||||
);
|
||||
})
|
||||
.toBe(true);
|
||||
|
||||
await browser.getByRole("button", { name: "Back" }).click();
|
||||
await browser.getByRole("button", { name: "Reload" }).click();
|
||||
await browser.getByRole("button", { name: "Developer tools" }).click();
|
||||
await expect
|
||||
.poll(async () => (await calls()).map(([name]) => name))
|
||||
.toEqual(expect.arrayContaining(["back", "reload", "openDevTools"]));
|
||||
|
||||
// App menus float above the page, so the native view steps aside while one is open.
|
||||
const before = (await calls()).length;
|
||||
await page.getByTestId("create-menu-trigger").click();
|
||||
await expect
|
||||
.poll(async () => (await calls()).slice(before).map(([name]) => name))
|
||||
.toContain("hide");
|
||||
await page.keyboard.press("Escape");
|
||||
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("show");
|
||||
|
||||
await browser.getByRole("button", { name: "Close panel" }).click();
|
||||
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("hide");
|
||||
});
|
||||
@ -75,7 +75,11 @@ test("local settings open and save integrations without an app session", async (
|
||||
});
|
||||
|
||||
test("ordinary browsers cannot use local settings", async ({ page }) => {
|
||||
await page.goto("/desktop-settings");
|
||||
await expect(page.getByRole("alert")).toContainText("Could not open local settings");
|
||||
await expect(async () => {
|
||||
await page.goto("/desktop-settings", { waitUntil: "domcontentloaded" });
|
||||
await expect(page.getByRole("alert")).toContainText("Could not open local settings", {
|
||||
timeout: 5_000,
|
||||
});
|
||||
}).toPass({ timeout: 45_000 });
|
||||
await expect(page.getByRole("button", { name: "Models", exact: true })).toBeHidden();
|
||||
});
|
||||
|
||||
@ -50,6 +50,16 @@ test("live page helper preserves identity, masks passwords, and reports partial
|
||||
expect(snapshot.title).toBe("Fixture");
|
||||
expect(snapshot.tree).toContain("Readable article content");
|
||||
expect(JSON.stringify(snapshot)).not.toContain("fake-test-password");
|
||||
// Element boxes are screen pixels for the visible bot cursor.
|
||||
expect(snapshot.screen.width).toBeGreaterThan(0);
|
||||
expect(snapshot.screen.height).toBeGreaterThan(0);
|
||||
const saveBox = snapshot.elements.find((element: { name: string }) => element.name === "Save")
|
||||
.box as number[];
|
||||
const saveRect = (await page.locator("#save").boundingBox())!;
|
||||
expect(saveBox[2]).toBe(Math.round(saveRect.width));
|
||||
expect(saveBox[3]).toBe(Math.round(saveRect.height));
|
||||
expect(saveBox[0]).toBeGreaterThanOrEqual(Math.round(saveRect.x));
|
||||
expect(saveBox[1]).toBeGreaterThanOrEqual(Math.round(saveRect.y));
|
||||
const ref = (snapshot: { elements: { name: string; ref: string }[] }, name: string) =>
|
||||
snapshot.elements.find((element) => element.name === name)!.ref;
|
||||
const result = await command("act", {
|
||||
|
||||
@ -3,6 +3,7 @@ import {
|
||||
captureScreenshot,
|
||||
completeOnboarding,
|
||||
openNewSpace,
|
||||
routePatchRpc,
|
||||
rpc,
|
||||
signup,
|
||||
unterhaltungenOnly,
|
||||
@ -45,6 +46,30 @@ test("Projekte + opens new project dialog instead of group form", async ({ page
|
||||
await expect(page.getByTestId("project-start-main")).toHaveText(/Starten/);
|
||||
});
|
||||
|
||||
for (const owner of [true, false]) {
|
||||
const role = owner ? "owner" : "member";
|
||||
const title = owner
|
||||
? "new project dialog offers a host folder to the deployment owner"
|
||||
: "new project dialog offers no host folder to a member";
|
||||
test(title, async ({ page }, testInfo) => {
|
||||
await signup(page, `proj-folder-${role}-${Date.now()}@rakazo.test`, "password12", "Folder");
|
||||
await completeOnboarding(page);
|
||||
// Whoever signs up first owns the deployment, which depends on spec order.
|
||||
await routePatchRpc(page, "bootstrap", (json) => ({
|
||||
...json,
|
||||
me: { ...(json.me as Record<string, unknown>), isDeploymentOwner: owner },
|
||||
}));
|
||||
await page.goto("/app");
|
||||
await page.waitForURL(/\/app\/[^/]+$/);
|
||||
|
||||
await page.locator(".sidebar-projekte").getByTestId("create-project-trigger").click();
|
||||
const dialog = page.getByRole("dialog", { name: "New project" });
|
||||
await expect(dialog.getByLabel("Name")).toBeVisible();
|
||||
await expect(dialog.getByLabel("Project folder")).toHaveCount(owner ? 1 : 0);
|
||||
await captureScreenshot(page, testInfo, `new-project-${role}`);
|
||||
});
|
||||
}
|
||||
|
||||
test("spaces stay invisible by default and chat creation requires approval", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
|
||||
@ -14,7 +14,6 @@ import {
|
||||
showSessionUnavailable,
|
||||
} from "./lib/session-gate";
|
||||
import { IntegrationSetupPage } from "./pages/IntegrationSetup";
|
||||
import { LocalSettingsPage } from "./pages/LocalSettings";
|
||||
import { McpOAuthCallbackPage } from "./pages/McpOAuthCallback";
|
||||
import { ShellPage } from "./pages/Shell";
|
||||
|
||||
@ -30,9 +29,18 @@ const OnboardingPage = lazyWithRecovery("pages/Onboarding", () =>
|
||||
const WelcomePage = lazyWithRecovery("pages/Welcome", () =>
|
||||
import("./pages/Welcome").then((module) => ({ default: module.WelcomePage })),
|
||||
);
|
||||
const LocalSettingsPage = lazyWithRecovery("pages/LocalSettings", () =>
|
||||
import("./pages/LocalSettings").then((module) => ({ default: module.LocalSettingsPage })),
|
||||
);
|
||||
|
||||
export function App() {
|
||||
if (window.location.pathname === LOCAL_SETTINGS_PAGE) return <LocalSettingsPage />;
|
||||
if (window.location.pathname === LOCAL_SETTINGS_PAGE) {
|
||||
return (
|
||||
<Suspense fallback={<div className="h-full bg-background" data-rakazo-app-state="loading" />}>
|
||||
<LocalSettingsPage />
|
||||
</Suspense>
|
||||
);
|
||||
}
|
||||
return <SessionApp />;
|
||||
}
|
||||
|
||||
|
||||
@ -4,13 +4,9 @@ import { Trans, useLingui } from "@lingui/react/macro";
|
||||
import { ChatMarkdown } from "@rakazo/chat-ui/web";
|
||||
import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web";
|
||||
import { Download, FileText, X } from "lucide-react";
|
||||
import { type RefObject, useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
type ArtifactTarget,
|
||||
downloadArtifact,
|
||||
downloadArtifactBytes,
|
||||
fetchArtifactBytes,
|
||||
} from "../lib/artifact-open";
|
||||
import { type RefObject, useMemo, useRef, useState } from "react";
|
||||
import { useArtifactBytes } from "../hooks/useArtifactBytes";
|
||||
import { type ArtifactTarget, downloadArtifact, downloadArtifactBytes } from "../lib/artifact-open";
|
||||
|
||||
type ArtifactFileCardProps = {
|
||||
target: ArtifactTarget;
|
||||
@ -18,6 +14,8 @@ type ArtifactFileCardProps = {
|
||||
name: string;
|
||||
mimeType: string;
|
||||
size: number;
|
||||
/** Shows the preview beside the conversation instead of in a dialog. */
|
||||
onOpenBeside?: () => void;
|
||||
};
|
||||
|
||||
export function ArtifactFileCard(props: ArtifactFileCardProps) {
|
||||
@ -63,7 +61,7 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
|
||||
ref={previewButton}
|
||||
type="button"
|
||||
aria-label={t`Preview ${props.name}`}
|
||||
onClick={() => setPreviewOpen(true)}
|
||||
onClick={() => (props.onOpenBeside ? props.onOpenBeside() : setPreviewOpen(true))}
|
||||
className="flex min-w-0 flex-1 items-center gap-3 px-4 py-3 text-left hover:bg-accent"
|
||||
>
|
||||
<span className="grid h-10 w-10 shrink-0 place-items-center rounded-[10px] bg-muted text-foreground">
|
||||
@ -111,39 +109,16 @@ function MarkdownPreview({
|
||||
}: ArtifactFileCardProps & { closeButtonRef: RefObject<HTMLButtonElement | null> }) {
|
||||
const { t } = useLingui();
|
||||
const [downloadError, setDownloadError] = useState<string | null>(null);
|
||||
const [state, setState] = useState<
|
||||
| { status: "loading" }
|
||||
| { status: "ready"; bytes: Uint8Array; markdown: string }
|
||||
| { status: "error"; message: string }
|
||||
>({ status: "loading" });
|
||||
const targetBotId = "botId" in target ? target.botId : undefined;
|
||||
const targetGroupId = "groupId" in target ? target.groupId : undefined;
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const artifactTarget: ArtifactTarget =
|
||||
targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! };
|
||||
void fetchArtifactBytes(artifactTarget, artifactId)
|
||||
.then((bytes) => {
|
||||
if (cancelled) return;
|
||||
try {
|
||||
const markdown = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
setState({ status: "ready", bytes, markdown });
|
||||
} catch {
|
||||
setState({ status: "error", message: t`This file is not valid UTF-8 Markdown.` });
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
if (cancelled) return;
|
||||
setState({
|
||||
status: "error",
|
||||
message: error instanceof Error ? error.message : t`Could not load this file.`,
|
||||
});
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [artifactId, targetBotId, targetGroupId, t]);
|
||||
const loaded = useArtifactBytes(target, artifactId);
|
||||
const state = useMemo(() => {
|
||||
if (loaded.status !== "ready") return loaded;
|
||||
try {
|
||||
const markdown = new TextDecoder("utf-8", { fatal: true }).decode(loaded.bytes);
|
||||
return { status: "ready" as const, bytes: loaded.bytes, markdown };
|
||||
} catch {
|
||||
return { status: "error" as const, message: t`This file is not valid UTF-8 Markdown.` };
|
||||
}
|
||||
}, [loaded, t]);
|
||||
|
||||
return (
|
||||
<>
|
||||
|
||||
165
apps/web/src/components/ArtifactPane.tsx
Normal file
165
apps/web/src/components/ArtifactPane.tsx
Normal file
@ -0,0 +1,165 @@
|
||||
import { Trans, useLingui } from "@lingui/react/macro";
|
||||
import { ChatMarkdown } from "@rakazo/chat-ui/web";
|
||||
import { isAttachmentImageMimeType } from "@rakazo/contracts";
|
||||
import type { WorkspaceArtifact } from "@rakazo/core";
|
||||
import { Button } from "@rakazo/ui-web";
|
||||
import { Download, X } from "lucide-react";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { useArtifactBytes } from "../hooks/useArtifactBytes";
|
||||
import { type ArtifactTarget, downloadArtifactBytes } from "../lib/artifact-open";
|
||||
import { parseCsvPreview } from "../lib/csv-preview";
|
||||
|
||||
/** A file the bot made, open beside the conversation so the user can read it while they talk. */
|
||||
export function ArtifactPane({
|
||||
target,
|
||||
artifact,
|
||||
onClose,
|
||||
}: {
|
||||
target: ArtifactTarget;
|
||||
artifact: WorkspaceArtifact;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const { t } = useLingui();
|
||||
const loaded = useArtifactBytes(target, artifact.artifactId);
|
||||
const { name, mimeType } = artifact;
|
||||
|
||||
return (
|
||||
<section
|
||||
data-testid="artifact-pane"
|
||||
aria-label={name}
|
||||
className="flex h-full min-h-0 w-full flex-col"
|
||||
>
|
||||
<header className="flex h-14 shrink-0 items-center gap-1 border-b border-border px-5">
|
||||
<h2 className="min-w-0 flex-1 truncate text-ui-body font-medium text-foreground" dir="auto">
|
||||
{name}
|
||||
</h2>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="text-muted-foreground"
|
||||
aria-label={t`Download ${name}`}
|
||||
title={t`Download ${name}`}
|
||||
disabled={loaded.status !== "ready"}
|
||||
onClick={() => {
|
||||
if (loaded.status === "ready") downloadArtifactBytes(name, mimeType, loaded.bytes);
|
||||
}}
|
||||
>
|
||||
<Download strokeWidth={1.8} className="size-icon-md" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="text-muted-foreground"
|
||||
aria-label={t`Close panel`}
|
||||
onClick={onClose}
|
||||
>
|
||||
<X strokeWidth={1.8} className="size-icon-md" />
|
||||
</Button>
|
||||
</header>
|
||||
<div className="rk-scroll min-h-0 flex-1 overflow-auto">
|
||||
{loaded.status === "loading" ? (
|
||||
<div className="px-6 py-8 text-ui-body text-muted-foreground">
|
||||
<Trans>Loading preview…</Trans>
|
||||
</div>
|
||||
) : loaded.status === "error" ? (
|
||||
<div
|
||||
role="alert"
|
||||
className="m-6 rounded-xl border border-destructive/40 bg-destructive/10 px-4 py-3 text-ui-body text-destructive"
|
||||
>
|
||||
{loaded.message}
|
||||
</div>
|
||||
) : isAttachmentImageMimeType(mimeType) ? (
|
||||
<PaneImage bytes={loaded.bytes} mimeType={mimeType} name={name} />
|
||||
) : (
|
||||
<PaneText bytes={loaded.bytes} mimeType={mimeType} />
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function PaneImage({
|
||||
bytes,
|
||||
mimeType,
|
||||
name,
|
||||
}: {
|
||||
bytes: Uint8Array;
|
||||
mimeType: string;
|
||||
name: string;
|
||||
}) {
|
||||
const [src, setSrc] = useState<string | null>(null);
|
||||
useEffect(() => {
|
||||
const url = URL.createObjectURL(new Blob([new Uint8Array(bytes)], { type: mimeType }));
|
||||
setSrc(url);
|
||||
return () => URL.revokeObjectURL(url);
|
||||
}, [bytes, mimeType]);
|
||||
return src ? (
|
||||
<img src={src} alt={name} className="mx-auto block max-w-full p-6" draggable={false} />
|
||||
) : null;
|
||||
}
|
||||
|
||||
function PaneText({ bytes, mimeType }: { bytes: Uint8Array; mimeType: string }) {
|
||||
const { t } = useLingui();
|
||||
const text = useMemo(() => new TextDecoder("utf-8").decode(bytes), [bytes]);
|
||||
if (mimeType === "text/markdown") {
|
||||
return (
|
||||
<article className="mx-auto w-full max-w-[760px] px-6 py-8 text-ui-body leading-7 text-foreground lg:px-10">
|
||||
<ChatMarkdown>{text}</ChatMarkdown>
|
||||
</article>
|
||||
);
|
||||
}
|
||||
if (mimeType === "text/csv") {
|
||||
const { rows, truncated } = parseCsvPreview(text);
|
||||
const [head, ...body] = rows;
|
||||
const shownRows = body.length;
|
||||
return (
|
||||
<div className="p-4">
|
||||
<table className="w-full border-collapse text-ui-body-sm text-foreground">
|
||||
{head ? (
|
||||
<thead>
|
||||
<tr>
|
||||
{head.map((cell, index) => (
|
||||
<th
|
||||
key={index}
|
||||
className="sticky top-0 border-b border-border bg-background px-3 py-2 text-start font-medium"
|
||||
>
|
||||
{cell}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
) : null}
|
||||
<tbody>
|
||||
{body.map((row, rowIndex) => (
|
||||
<tr key={rowIndex} className="border-b border-border/60">
|
||||
{row.map((cell, index) => (
|
||||
<td key={index} className="px-3 py-1.5 align-top whitespace-pre-wrap">
|
||||
{cell}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{truncated ? (
|
||||
<p className="px-3 pt-3 text-ui-body-sm text-muted-foreground">
|
||||
{t`First ${shownRows} rows`}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<pre className="whitespace-pre-wrap break-words px-6 py-6 font-mono text-ui-body-sm text-foreground">
|
||||
{mimeType === "application/json" ? prettyJson(text) : text}
|
||||
</pre>
|
||||
);
|
||||
}
|
||||
|
||||
function prettyJson(text: string): string {
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(text), null, 2);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
@ -87,6 +87,7 @@ export function AskCard({
|
||||
|
||||
return (
|
||||
<div
|
||||
data-ask-card
|
||||
data-testid={secretInput ? "secret-ask-card" : undefined}
|
||||
className={cn("max-w-[74%] rounded-2xl border border-border bg-card px-5 py-4", className)}
|
||||
>
|
||||
|
||||
317
apps/web/src/components/browser/InAppBrowser.tsx
Normal file
317
apps/web/src/components/browser/InAppBrowser.tsx
Normal file
@ -0,0 +1,317 @@
|
||||
import { useLingui } from "@lingui/react/macro";
|
||||
import type {
|
||||
RakazoDesktopBrowserPreview,
|
||||
RakazoDesktopBrowserPreviewState,
|
||||
} from "@rakazo/contracts";
|
||||
import { Button, cn, Input } from "@rakazo/ui-web";
|
||||
import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react";
|
||||
import { type FormEvent, useCallback, useEffect, useRef, useState } from "react";
|
||||
import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url";
|
||||
import { desktopBridge } from "../../lib/desktop";
|
||||
|
||||
const LAST_URL_KEY = "rk:in-app-browser:url";
|
||||
|
||||
function readLastUrl(): string {
|
||||
try {
|
||||
return window.localStorage.getItem(LAST_URL_KEY) ?? "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function rememberUrl(url: string) {
|
||||
try {
|
||||
window.localStorage.setItem(LAST_URL_KEY, url);
|
||||
} catch {
|
||||
/* A per-viewer convenience only. */
|
||||
}
|
||||
}
|
||||
|
||||
const EMPTY_STATE: RakazoDesktopBrowserPreviewState = {
|
||||
url: "",
|
||||
title: "",
|
||||
loading: false,
|
||||
canGoBack: false,
|
||||
canGoForward: false,
|
||||
error: null,
|
||||
};
|
||||
|
||||
/** DOM layers the native desktop view would otherwise paint over. */
|
||||
const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]';
|
||||
|
||||
/**
|
||||
* Address bar plus page. On desktop the page is a native Chromium view laid over the reserved
|
||||
* area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe.
|
||||
*/
|
||||
export function InAppBrowser({ onClose }: { onClose: () => void }) {
|
||||
const { t } = useLingui();
|
||||
const native = desktopBridge()?.browserPreview;
|
||||
const [state, setState] = useState<RakazoDesktopBrowserPreviewState>(() => ({
|
||||
...EMPTY_STATE,
|
||||
url: native ? "" : readLastUrl(),
|
||||
}));
|
||||
const [draft, setDraft] = useState(state.url);
|
||||
const [invalid, setInvalid] = useState(false);
|
||||
const [frameKey, setFrameKey] = useState(0);
|
||||
const editing = useRef(false);
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!native) return;
|
||||
let live = true;
|
||||
const apply = (next: RakazoDesktopBrowserPreviewState) => {
|
||||
if (!live) return;
|
||||
setState(next);
|
||||
if (!editing.current) setDraft(next.url);
|
||||
};
|
||||
const unsubscribe = native.onState(apply);
|
||||
void native
|
||||
.state()
|
||||
.then((current) => {
|
||||
apply(current);
|
||||
const last = readLastUrl();
|
||||
if (!current.url && last) void native.navigate(last).catch(() => undefined);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
return () => {
|
||||
live = false;
|
||||
unsubscribe();
|
||||
};
|
||||
}, [native]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!state.url) inputRef.current?.focus();
|
||||
}, [state.url]);
|
||||
|
||||
const go = useCallback(
|
||||
(input: string) => {
|
||||
const url = normalizeBrowserUrl(input);
|
||||
setInvalid(!url);
|
||||
if (!url) return;
|
||||
editing.current = false;
|
||||
setDraft(url);
|
||||
rememberUrl(url);
|
||||
if (native) {
|
||||
void native.navigate(url).catch(() => setInvalid(true));
|
||||
} else {
|
||||
setState((current) => ({ ...current, url, loading: true, error: null }));
|
||||
setFrameKey((key) => key + 1);
|
||||
}
|
||||
},
|
||||
[native],
|
||||
);
|
||||
|
||||
const reload = useCallback(() => {
|
||||
if (native) {
|
||||
void (state.loading ? native.stop() : native.reload()).catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
if (!state.url) return;
|
||||
setState((current) => ({ ...current, loading: true }));
|
||||
setFrameKey((key) => key + 1);
|
||||
}, [native, state.loading, state.url]);
|
||||
|
||||
const onSubmit = (event: FormEvent) => {
|
||||
event.preventDefault();
|
||||
go(draft);
|
||||
inputRef.current?.blur();
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col" data-testid="in-app-browser">
|
||||
<form onSubmit={onSubmit} className="relative flex items-center gap-0.5 border-b px-2 py-1.5">
|
||||
{native ? (
|
||||
<>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Back`}
|
||||
disabled={!state.canGoBack}
|
||||
onClick={() => void native.back().catch(() => undefined)}
|
||||
>
|
||||
<ArrowLeft strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Forward`}
|
||||
disabled={!state.canGoForward}
|
||||
onClick={() => void native.forward().catch(() => undefined)}
|
||||
>
|
||||
<ArrowRight strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
</>
|
||||
) : null}
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={state.loading ? t`Stop` : t`Reload`}
|
||||
disabled={!state.url}
|
||||
onClick={reload}
|
||||
>
|
||||
{state.loading && native ? (
|
||||
<X strokeWidth={1.8} className="size-icon-md" />
|
||||
) : (
|
||||
<RotateCw strokeWidth={1.7} className="size-icon-md" />
|
||||
)}
|
||||
</Button>
|
||||
<Input
|
||||
ref={inputRef}
|
||||
value={draft}
|
||||
aria-label={t`Address`}
|
||||
aria-invalid={invalid || undefined}
|
||||
inputMode="url"
|
||||
autoComplete="off"
|
||||
autoCapitalize="off"
|
||||
spellCheck={false}
|
||||
placeholder="localhost:5173"
|
||||
onFocus={(event) => {
|
||||
editing.current = true;
|
||||
event.currentTarget.select();
|
||||
}}
|
||||
onBlur={() => {
|
||||
editing.current = false;
|
||||
}}
|
||||
onChange={(event) => {
|
||||
setDraft(event.target.value);
|
||||
setInvalid(false);
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key !== "Escape") return;
|
||||
setDraft(state.url);
|
||||
setInvalid(false);
|
||||
event.currentTarget.blur();
|
||||
}}
|
||||
className="mx-1 h-control-sm flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm"
|
||||
/>
|
||||
{native ? (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Developer tools`}
|
||||
disabled={!state.url}
|
||||
onClick={() => void native.openDevTools().catch(() => undefined)}
|
||||
>
|
||||
<Code strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
) : null}
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Open in browser`}
|
||||
disabled={!state.url}
|
||||
onClick={() => window.open(state.url, "_blank", "noopener,noreferrer")}
|
||||
>
|
||||
<ExternalLink strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Close panel`}
|
||||
onClick={onClose}
|
||||
>
|
||||
<X strokeWidth={1.8} className="size-icon-md" />
|
||||
</Button>
|
||||
<div
|
||||
aria-hidden
|
||||
className={cn(
|
||||
"absolute inset-x-0 -bottom-px h-0.5 overflow-hidden transition-opacity duration-300",
|
||||
state.loading ? "opacity-100" : "opacity-0",
|
||||
)}
|
||||
>
|
||||
<div className="rk-browser-progress h-full w-1/3 rounded-full bg-primary/70" />
|
||||
</div>
|
||||
</form>
|
||||
<div className="relative min-h-0 flex-1 bg-background">
|
||||
{native ? (
|
||||
<NativeViewport browser={native} visible={Boolean(state.url) && !state.error} />
|
||||
) : state.url ? (
|
||||
<iframe
|
||||
key={frameKey}
|
||||
title={state.title || t`Page`}
|
||||
src={state.url}
|
||||
sandbox={browserFrameSandbox(state.url, window.location.origin)}
|
||||
allow="clipboard-read; clipboard-write; fullscreen"
|
||||
onLoad={() => setState((current) => ({ ...current, loading: false }))}
|
||||
className="h-full w-full border-0 bg-background"
|
||||
data-testid="in-app-browser-frame"
|
||||
/>
|
||||
) : null}
|
||||
{state.error ? (
|
||||
<div
|
||||
role="alert"
|
||||
className="absolute inset-0 grid place-items-center px-6 text-center text-ui-body-sm text-muted-foreground"
|
||||
>
|
||||
{state.error}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserves the page area and keeps the native view glued to it while the panel resizes,
|
||||
* hiding it whenever app UI (dialogs, menus) floats where the view would paint over it.
|
||||
*/
|
||||
function NativeViewport({
|
||||
browser,
|
||||
visible,
|
||||
}: {
|
||||
browser: RakazoDesktopBrowserPreview;
|
||||
visible: boolean;
|
||||
}) {
|
||||
const ref = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const element = ref.current;
|
||||
if (!element) return;
|
||||
let last = "";
|
||||
let frame = 0;
|
||||
const sync = () => {
|
||||
frame = 0;
|
||||
const rect = element.getBoundingClientRect();
|
||||
const covered = Array.from(document.querySelectorAll(FLOATING_UI)).some(
|
||||
(node) => !element.contains(node),
|
||||
);
|
||||
const show = visible && rect.width >= 1 && rect.height >= 1 && !covered;
|
||||
const bounds = {
|
||||
x: Math.round(rect.left),
|
||||
y: Math.round(rect.top),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
const key = show ? JSON.stringify(bounds) : "hidden";
|
||||
if (key === last) return;
|
||||
last = key;
|
||||
void (show ? browser.show(bounds) : browser.hide()).catch(() => undefined);
|
||||
};
|
||||
const schedule = () => {
|
||||
if (!frame) frame = requestAnimationFrame(sync);
|
||||
};
|
||||
const resize = new ResizeObserver(schedule);
|
||||
resize.observe(element);
|
||||
const floating = new MutationObserver(schedule);
|
||||
floating.observe(document.body, { childList: true, subtree: true });
|
||||
window.addEventListener("resize", schedule);
|
||||
// Sibling layout (sidebars, transitions) can move the area without resizing it.
|
||||
const settle = setInterval(schedule, 250);
|
||||
schedule();
|
||||
return () => {
|
||||
cancelAnimationFrame(frame);
|
||||
resize.disconnect();
|
||||
floating.disconnect();
|
||||
window.removeEventListener("resize", schedule);
|
||||
clearInterval(settle);
|
||||
void browser.hide().catch(() => undefined);
|
||||
};
|
||||
}, [browser, visible]);
|
||||
|
||||
return <div ref={ref} className="absolute inset-0" data-testid="in-app-browser-viewport" />;
|
||||
}
|
||||
165
apps/web/src/components/computer/AgentCursor.tsx
Normal file
165
apps/web/src/components/computer/AgentCursor.tsx
Normal file
@ -0,0 +1,165 @@
|
||||
import { screenPointInViewer } from "@rakazo/core";
|
||||
import { Keyboard } from "lucide-react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
agentPointerFeed,
|
||||
POINTER_IDLE_MS,
|
||||
type PointerPoint,
|
||||
type PointerScreen,
|
||||
type PointerSegment,
|
||||
planPointerTimeline,
|
||||
pointerFrameAt,
|
||||
} from "../../lib/agent-pointer";
|
||||
|
||||
function prefersReducedMotion() {
|
||||
return (
|
||||
typeof window.matchMedia === "function" &&
|
||||
window.matchMedia("(prefers-reduced-motion: reduce)").matches
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Draws a bot's announced pointer over its live screen: it glides to each target, ripples on
|
||||
* clicks and shows when it types. Frames are written straight to the DOM so the chat never
|
||||
* re-renders at animation rate.
|
||||
*/
|
||||
export function AgentCursor({
|
||||
botId,
|
||||
color,
|
||||
name,
|
||||
}: {
|
||||
botId: string;
|
||||
color: string;
|
||||
name: string;
|
||||
}) {
|
||||
const layerRef = useRef<HTMLDivElement>(null);
|
||||
const cursorRef = useRef<HTMLDivElement>(null);
|
||||
const rippleRef = useRef<HTMLSpanElement>(null);
|
||||
const [typing, setTyping] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const layer = layerRef.current;
|
||||
const cursor = cursorRef.current;
|
||||
const ripple = rippleRef.current;
|
||||
if (!layer || !cursor || !ripple) return;
|
||||
const reducedMotion = prefersReducedMotion();
|
||||
let screen: PointerScreen | null = null;
|
||||
let segments: PointerSegment[] = [];
|
||||
let position: PointerPoint | null = null;
|
||||
let frame = 0;
|
||||
let idleTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let viewer = { width: layer.clientWidth, height: layer.clientHeight };
|
||||
let shownTyping = false;
|
||||
|
||||
const place = (point: PointerPoint, action: string | null, progress: number) => {
|
||||
if (!screen) return;
|
||||
const projected = screenPointInViewer(point, viewer, screen);
|
||||
cursor.style.transform = `translate3d(${projected.x}px, ${projected.y}px, 0)`;
|
||||
cursor.dataset.action = action ?? "move";
|
||||
const pressing = action === "click" || action === "double_click" || action === "right_click";
|
||||
const pulse = action === "double_click" ? (progress * 2) % 1 : progress;
|
||||
ripple.style.opacity = pressing && !reducedMotion ? String(0.55 * (1 - pulse)) : "0";
|
||||
ripple.style.transform = `scale(${0.35 + pulse * 1.15})`;
|
||||
cursor.style.setProperty("--agent-cursor-press", pressing && pulse < 0.35 ? "0.86" : "1");
|
||||
const nextTyping = action === "type" || action === "key";
|
||||
if (nextTyping !== shownTyping) {
|
||||
shownTyping = nextTyping;
|
||||
setTyping(nextTyping);
|
||||
}
|
||||
};
|
||||
|
||||
const rest = () => {
|
||||
clearTimeout(idleTimer);
|
||||
idleTimer = setTimeout(() => {
|
||||
cursor.style.opacity = "0";
|
||||
}, POINTER_IDLE_MS);
|
||||
};
|
||||
|
||||
const tick = () => {
|
||||
frame = 0;
|
||||
const now = performance.now();
|
||||
const state = pointerFrameAt(segments, now, reducedMotion);
|
||||
if (!state) return;
|
||||
position = state.point;
|
||||
place(state.point, state.action?.kind ?? null, state.action?.progress ?? 0);
|
||||
if (state.done) {
|
||||
segments = [];
|
||||
rest();
|
||||
return;
|
||||
}
|
||||
frame = requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
const unsubscribe = agentPointerFeed.subscribe(botId, (payload) => {
|
||||
const now = performance.now();
|
||||
const current = pointerFrameAt(segments, now, reducedMotion)?.point ?? position;
|
||||
const sameScreen =
|
||||
screen?.width === payload.screen.width && screen?.height === payload.screen.height;
|
||||
screen = payload.screen;
|
||||
// A first announcement starts at its target rather than flying in from a corner.
|
||||
const firstTarget = payload.steps.find((step) => typeof step.x === "number");
|
||||
const from =
|
||||
current && sameScreen
|
||||
? current
|
||||
: firstTarget
|
||||
? { x: firstTarget.x ?? 0, y: firstTarget.y ?? 0 }
|
||||
: { x: payload.screen.width / 2, y: payload.screen.height / 2 };
|
||||
segments = planPointerTimeline(from, payload.steps, now);
|
||||
clearTimeout(idleTimer);
|
||||
cursor.style.opacity = "1";
|
||||
if (!frame) frame = requestAnimationFrame(tick);
|
||||
});
|
||||
|
||||
const resize = new ResizeObserver(() => {
|
||||
viewer = { width: layer.clientWidth, height: layer.clientHeight };
|
||||
if (position && !frame) place(position, null, 0);
|
||||
});
|
||||
resize.observe(layer);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
resize.disconnect();
|
||||
cancelAnimationFrame(frame);
|
||||
clearTimeout(idleTimer);
|
||||
};
|
||||
}, [botId]);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={layerRef}
|
||||
aria-hidden
|
||||
data-testid="agent-cursor-layer"
|
||||
className="pointer-events-none absolute inset-0 overflow-hidden"
|
||||
>
|
||||
<div
|
||||
ref={cursorRef}
|
||||
data-testid="agent-cursor"
|
||||
className="absolute top-0 left-0 opacity-0 transition-opacity duration-300 will-change-transform"
|
||||
style={{ color }}
|
||||
>
|
||||
<span
|
||||
ref={rippleRef}
|
||||
className="absolute -top-5 -left-5 size-10 rounded-full border-2 border-current opacity-0"
|
||||
/>
|
||||
<svg
|
||||
viewBox="0 0 20 22"
|
||||
className="relative block size-5 origin-top-left drop-shadow-md transition-transform duration-100"
|
||||
style={{ transform: "scale(var(--agent-cursor-press, 1))" }}
|
||||
>
|
||||
<path
|
||||
d="M1.5 1.5 L1.5 17.5 L6 13.2 L9.2 20.2 L12.4 18.8 L9.3 12 L15.5 12 Z"
|
||||
fill="currentColor"
|
||||
className="stroke-background"
|
||||
strokeWidth="1.6"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
<span className="absolute top-5 left-3.5 flex max-w-40 items-center gap-1 rounded-full bg-popover px-2 py-0.5 text-ui-label font-medium whitespace-nowrap text-popover-foreground shadow-sm ring-1 ring-border">
|
||||
<span className="size-1.5 shrink-0 rounded-full bg-current" style={{ color }} />
|
||||
<span className="truncate">{name}</span>
|
||||
{typing ? <Keyboard strokeWidth={1.8} className="size-3 shrink-0" /> : null}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
38
apps/web/src/hooks/useArtifactBytes.ts
Normal file
38
apps/web/src/hooks/useArtifactBytes.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import { useLingui } from "@lingui/react/macro";
|
||||
import { useEffect, useState } from "react";
|
||||
import { type ArtifactTarget, fetchArtifactBytes } from "../lib/artifact-open";
|
||||
|
||||
export type ArtifactBytesState =
|
||||
| { status: "loading" }
|
||||
| { status: "ready"; bytes: Uint8Array }
|
||||
| { status: "error"; message: string };
|
||||
|
||||
export function useArtifactBytes(target: ArtifactTarget, artifactId: string): ArtifactBytesState {
|
||||
const { t } = useLingui();
|
||||
const [state, setState] = useState<ArtifactBytesState>({ status: "loading" });
|
||||
const targetBotId = "botId" in target ? target.botId : undefined;
|
||||
const targetGroupId = "groupId" in target ? target.groupId : undefined;
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setState({ status: "loading" });
|
||||
const artifactTarget: ArtifactTarget =
|
||||
targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! };
|
||||
void fetchArtifactBytes(artifactTarget, artifactId)
|
||||
.then((bytes) => {
|
||||
if (!cancelled) setState({ status: "ready", bytes });
|
||||
})
|
||||
.catch((error) => {
|
||||
if (cancelled) return;
|
||||
setState({
|
||||
status: "error",
|
||||
message: error instanceof Error ? error.message : t`Could not load this file.`,
|
||||
});
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [artifactId, targetBotId, targetGroupId, t]);
|
||||
|
||||
return state;
|
||||
}
|
||||
@ -1,13 +1,20 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { readPaneWidth, writePaneWidth } from "../lib/workspace-pane-pref";
|
||||
|
||||
export function useSidebarResize(
|
||||
initialWidth: number,
|
||||
side: "left" | "right",
|
||||
minWidth: number,
|
||||
maxWidth: number,
|
||||
/** Remembers the width across reloads on this device. */
|
||||
storageKey?: string,
|
||||
) {
|
||||
const [width, setWidth] = useState(initialWidth);
|
||||
const [width, setWidth] = useState(() =>
|
||||
storageKey ? readPaneWidth(storageKey, initialWidth, minWidth, maxWidth) : initialWidth,
|
||||
);
|
||||
const [isResizing, setIsResizing] = useState(false);
|
||||
const widthRef = useRef(width);
|
||||
widthRef.current = width;
|
||||
|
||||
useEffect(() => {
|
||||
if (!isResizing) return;
|
||||
@ -26,6 +33,7 @@ export function useSidebarResize(
|
||||
const handlePointerUp = () => {
|
||||
setIsResizing(false);
|
||||
document.body.style.cursor = "default";
|
||||
if (storageKey) writePaneWidth(storageKey, widthRef.current);
|
||||
};
|
||||
|
||||
window.addEventListener("pointermove", handlePointerMove);
|
||||
@ -38,7 +46,7 @@ export function useSidebarResize(
|
||||
window.removeEventListener("pointerup", handlePointerUp);
|
||||
document.body.style.cursor = "default";
|
||||
};
|
||||
}, [isResizing, side, minWidth, maxWidth]);
|
||||
}, [isResizing, side, minWidth, maxWidth, storageKey]);
|
||||
|
||||
const startResizing = useCallback((e: React.PointerEvent) => {
|
||||
e.preventDefault();
|
||||
|
||||
16
apps/web/src/hooks/useWideLayout.ts
Normal file
16
apps/web/src/hooks/useWideLayout.ts
Normal file
@ -0,0 +1,16 @@
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
/** Wide enough to keep the conversation readable beside a working pane. */
|
||||
const WIDE_LAYOUT_QUERY = "(min-width: 1024px)";
|
||||
|
||||
export function useWideLayout(): boolean {
|
||||
const [wide, setWide] = useState(() => window.matchMedia(WIDE_LAYOUT_QUERY).matches);
|
||||
useEffect(() => {
|
||||
const media = window.matchMedia(WIDE_LAYOUT_QUERY);
|
||||
const sync = () => setWide(media.matches);
|
||||
sync();
|
||||
media.addEventListener("change", sync);
|
||||
return () => media.removeEventListener("change", sync);
|
||||
}, []);
|
||||
return wide;
|
||||
}
|
||||
88
apps/web/src/lib/agent-pointer.test.ts
Normal file
88
apps/web/src/lib/agent-pointer.test.ts
Normal file
@ -0,0 +1,88 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
agentPointerFeed,
|
||||
moveDurationMs,
|
||||
planPointerTimeline,
|
||||
pointerFrameAt,
|
||||
} from "./agent-pointer";
|
||||
|
||||
const payload = {
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [{ action: "click", x: 100, y: 100 }],
|
||||
};
|
||||
|
||||
describe("agentPointerFeed", () => {
|
||||
it("delivers valid live announcements only to the bot's viewers", () => {
|
||||
const seen: unknown[] = [];
|
||||
const other: unknown[] = [];
|
||||
const stop = agentPointerFeed.subscribe("bot-a", (value) => seen.push(value));
|
||||
const stopOther = agentPointerFeed.subscribe("bot-b", (value) => other.push(value));
|
||||
try {
|
||||
const now = Date.parse("2026-01-01T00:00:10Z");
|
||||
expect(agentPointerFeed.publish("bot-a", payload, "2026-01-01T00:00:05Z", now)).toBe(true);
|
||||
// Replayed history and malformed payloads never move the cursor.
|
||||
expect(agentPointerFeed.publish("bot-a", payload, "2025-12-31T23:00:00Z", now)).toBe(false);
|
||||
expect(agentPointerFeed.publish("bot-a", { ...payload, steps: [] }, undefined, now)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(agentPointerFeed.publish("bot-a", { kind: "status" }, undefined, now)).toBe(false);
|
||||
expect(seen).toEqual([payload]);
|
||||
expect(other).toEqual([]);
|
||||
} finally {
|
||||
stop();
|
||||
stopOther();
|
||||
}
|
||||
expect(agentPointerFeed.publish("bot-a", payload)).toBe(true);
|
||||
expect(seen).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pointer timeline", () => {
|
||||
it("travels, acts in place for keyboard steps, then rests", () => {
|
||||
const segments = planPointerTimeline(
|
||||
{ x: 0, y: 0 },
|
||||
[{ action: "click", x: 300, y: 400 }, { action: "type" }, { action: "move", x: 300, y: 400 }],
|
||||
1_000,
|
||||
);
|
||||
expect(segments.map((segment) => segment.to)).toEqual([
|
||||
{ x: 300, y: 400 },
|
||||
{ x: 300, y: 400 },
|
||||
{ x: 300, y: 400 },
|
||||
]);
|
||||
const [click, type, move] = segments;
|
||||
expect(click!.arrive - click!.start).toBe(moveDurationMs(500));
|
||||
expect(type!.arrive).toBe(type!.start);
|
||||
expect(move!.end).toBe(move!.start);
|
||||
|
||||
const midway = pointerFrameAt(segments, (click!.start + click!.arrive) / 2)!;
|
||||
expect(midway.action).toBeNull();
|
||||
// The bow lifts the cursor off the straight line between the points.
|
||||
const onLine = { x: 150, y: 200 };
|
||||
expect(Math.hypot(midway.point.x - onLine.x, midway.point.y - onLine.y)).toBeGreaterThan(10);
|
||||
|
||||
const clicking = pointerFrameAt(segments, click!.arrive + 1)!;
|
||||
expect(clicking.point).toEqual({ x: 300, y: 400 });
|
||||
expect(clicking.action?.kind).toBe("click");
|
||||
|
||||
const typing = pointerFrameAt(segments, type!.arrive + 10)!;
|
||||
expect(typing.action?.kind).toBe("type");
|
||||
|
||||
expect(pointerFrameAt(segments, move!.end + 1)).toEqual({
|
||||
point: { x: 300, y: 400 },
|
||||
action: null,
|
||||
done: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("jumps straight to the target when motion is reduced", () => {
|
||||
const segments = planPointerTimeline({ x: 0, y: 0 }, [{ action: "move", x: 50, y: 60 }], 0);
|
||||
expect(pointerFrameAt(segments, 1, true)?.point).toEqual({ x: 50, y: 60 });
|
||||
});
|
||||
|
||||
it("bounds travel time so long and short moves both feel responsive", () => {
|
||||
expect(moveDurationMs(0)).toBe(0);
|
||||
expect(moveDurationMs(5)).toBe(180);
|
||||
expect(moveDurationMs(10_000)).toBe(650);
|
||||
});
|
||||
});
|
||||
156
apps/web/src/lib/agent-pointer.ts
Normal file
156
apps/web/src/lib/agent-pointer.ts
Normal file
@ -0,0 +1,156 @@
|
||||
import {
|
||||
type ComputerPointerAction,
|
||||
ComputerPointerPayload,
|
||||
type ComputerPointerStep,
|
||||
} from "@rakazo/contracts";
|
||||
import { POINTER_EVENT_MAX_AGE_MS } from "@rakazo/core";
|
||||
|
||||
export type PointerPoint = { x: number; y: number };
|
||||
export type PointerScreen = ComputerPointerPayload["screen"];
|
||||
|
||||
export { POINTER_EVENT_MAX_AGE_MS };
|
||||
export const POINTER_IDLE_MS = 3_500;
|
||||
|
||||
type Listener = (payload: ComputerPointerPayload) => void;
|
||||
|
||||
/**
|
||||
* Live pointer announcements per bot. The thread subscription publishes `thread.computer`
|
||||
* pointer events here; screen viewers subscribe for the bot they show.
|
||||
*/
|
||||
class AgentPointerFeed {
|
||||
private readonly listeners = new Map<string, Set<Listener>>();
|
||||
|
||||
publish(botId: string, payload: unknown, createdAt?: string, now = Date.now()): boolean {
|
||||
const parsed = ComputerPointerPayload.safeParse(payload);
|
||||
if (!parsed.success) return false;
|
||||
const at = createdAt ? Date.parse(createdAt) : Number.NaN;
|
||||
if (Number.isFinite(at) && now - at > POINTER_EVENT_MAX_AGE_MS) return false;
|
||||
for (const listener of this.listeners.get(botId) ?? []) listener(parsed.data);
|
||||
return true;
|
||||
}
|
||||
|
||||
subscribe(botId: string, listener: Listener): () => void {
|
||||
let set = this.listeners.get(botId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
this.listeners.set(botId, set);
|
||||
}
|
||||
set.add(listener);
|
||||
return () => {
|
||||
set.delete(listener);
|
||||
if (set.size === 0) this.listeners.delete(botId);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const agentPointerFeed = new AgentPointerFeed();
|
||||
|
||||
export type PointerSegment = {
|
||||
from: PointerPoint;
|
||||
to: PointerPoint;
|
||||
/** Perpendicular bow in screen pixels so long moves read as a hand, not a teleport. */
|
||||
bow: number;
|
||||
start: number;
|
||||
arrive: number;
|
||||
end: number;
|
||||
action: ComputerPointerAction;
|
||||
};
|
||||
|
||||
const ACTION_DWELL_MS: Record<ComputerPointerAction, number> = {
|
||||
move: 0,
|
||||
click: 320,
|
||||
double_click: 440,
|
||||
right_click: 320,
|
||||
down: 160,
|
||||
up: 160,
|
||||
type: 700,
|
||||
key: 280,
|
||||
scroll: 420,
|
||||
};
|
||||
|
||||
export function moveDurationMs(distance: number): number {
|
||||
return distance < 1 ? 0 : Math.round(Math.min(650, Math.max(180, 140 + distance * 0.45)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Lays announced steps on a timeline that starts where the cursor currently is. Keyboard
|
||||
* steps carry no coordinates and act in place.
|
||||
*/
|
||||
export function planPointerTimeline(
|
||||
from: PointerPoint,
|
||||
steps: readonly ComputerPointerStep[],
|
||||
startAt: number,
|
||||
): PointerSegment[] {
|
||||
const segments: PointerSegment[] = [];
|
||||
let position = from;
|
||||
let time = startAt;
|
||||
for (const step of steps) {
|
||||
const to =
|
||||
typeof step.x === "number" && typeof step.y === "number"
|
||||
? { x: step.x, y: step.y }
|
||||
: position;
|
||||
const distance = Math.hypot(to.x - position.x, to.y - position.y);
|
||||
const arrive = time + moveDurationMs(distance);
|
||||
const end = arrive + ACTION_DWELL_MS[step.action];
|
||||
segments.push({
|
||||
from: position,
|
||||
to,
|
||||
bow: Math.min(48, distance * 0.12),
|
||||
start: time,
|
||||
arrive,
|
||||
end,
|
||||
action: step.action,
|
||||
});
|
||||
position = to;
|
||||
time = end;
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
|
||||
export type PointerFrame = {
|
||||
point: PointerPoint;
|
||||
/** The action being performed at the target, with 0..1 progress; null while travelling. */
|
||||
action: { kind: ComputerPointerAction; progress: number } | null;
|
||||
done: boolean;
|
||||
};
|
||||
|
||||
function easeInOutCubic(t: number): number {
|
||||
return t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2;
|
||||
}
|
||||
|
||||
export function pointerFrameAt(
|
||||
segments: readonly PointerSegment[],
|
||||
now: number,
|
||||
reducedMotion = false,
|
||||
): PointerFrame | null {
|
||||
if (segments.length === 0) return null;
|
||||
const last = segments[segments.length - 1]!;
|
||||
if (now >= last.end) return { point: last.to, action: null, done: true };
|
||||
const segment = segments.find((candidate) => now < candidate.end) ?? last;
|
||||
if (now < segment.arrive) {
|
||||
if (reducedMotion) return { point: segment.to, action: null, done: false };
|
||||
const span = segment.arrive - segment.start;
|
||||
const t = easeInOutCubic(span > 0 ? Math.max(0, (now - segment.start) / span) : 1);
|
||||
const dx = segment.to.x - segment.from.x;
|
||||
const dy = segment.to.y - segment.from.y;
|
||||
const length = Math.hypot(dx, dy) || 1;
|
||||
const lift = Math.sin(Math.PI * t) * segment.bow;
|
||||
return {
|
||||
point: {
|
||||
x: segment.from.x + dx * t - (dy / length) * lift,
|
||||
y: segment.from.y + dy * t + (dx / length) * lift,
|
||||
},
|
||||
action: null,
|
||||
done: false,
|
||||
};
|
||||
}
|
||||
const dwell = segment.end - segment.arrive;
|
||||
return {
|
||||
point: segment.to,
|
||||
action:
|
||||
segment.action === "move"
|
||||
? null
|
||||
: { kind: segment.action, progress: dwell > 0 ? (now - segment.arrive) / dwell : 1 },
|
||||
done: false,
|
||||
};
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { browserPreviewActivity } from "./browser-preview-context";
|
||||
|
||||
describe("browserPreviewActivity", () => {
|
||||
it("publishes cursors to subscribers and drops cleared ids", () => {
|
||||
for (const id of browserPreviewActivity.snapshot().keys()) browserPreviewActivity.clear(id);
|
||||
|
||||
const seen: string[][] = [];
|
||||
const unsubscribe = browserPreviewActivity.subscribe((activities) => {
|
||||
seen.push([...activities.keys()]);
|
||||
});
|
||||
|
||||
try {
|
||||
browserPreviewActivity.set("bot-a", 50, 40, "Ada");
|
||||
browserPreviewActivity.set("bot-a", 50, 40, "Ada");
|
||||
browserPreviewActivity.set("bot-b", 10, 20, "Bob");
|
||||
expect(browserPreviewActivity.snapshot().get("bot-a")).toEqual({
|
||||
x: 50,
|
||||
y: 40,
|
||||
label: "Ada",
|
||||
});
|
||||
|
||||
browserPreviewActivity.clear("bot-a");
|
||||
browserPreviewActivity.clear("missing");
|
||||
|
||||
expect(seen).toEqual([[], ["bot-a"], ["bot-a", "bot-b"], ["bot-b"]]);
|
||||
} finally {
|
||||
unsubscribe();
|
||||
for (const id of [...browserPreviewActivity.snapshot().keys()]) {
|
||||
browserPreviewActivity.clear(id);
|
||||
}
|
||||
}
|
||||
|
||||
expect(browserPreviewActivity.snapshot().size).toBe(0);
|
||||
});
|
||||
});
|
||||
@ -1,47 +0,0 @@
|
||||
export type BrowserPreviewCursor = {
|
||||
x: number;
|
||||
y: number;
|
||||
label: string;
|
||||
};
|
||||
|
||||
type Listener = (activities: ReadonlyMap<string, BrowserPreviewCursor>) => void;
|
||||
|
||||
/**
|
||||
* In-memory cursors for the desktop browser-preview panel.
|
||||
* The shell publishes working bots here; the panel subscribes.
|
||||
*/
|
||||
class BrowserPreviewActivity {
|
||||
private readonly activities = new Map<string, BrowserPreviewCursor>();
|
||||
private readonly listeners = new Set<Listener>();
|
||||
|
||||
subscribe(listener: Listener): () => void {
|
||||
this.listeners.add(listener);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): ReadonlyMap<string, BrowserPreviewCursor> {
|
||||
return new Map(this.activities);
|
||||
}
|
||||
|
||||
set(id: string, x: number, y: number, label: string) {
|
||||
const current = this.activities.get(id);
|
||||
if (current && current.x === x && current.y === y && current.label === label) return;
|
||||
this.activities.set(id, { x, y, label });
|
||||
this.emit();
|
||||
}
|
||||
|
||||
clear(id: string) {
|
||||
if (!this.activities.delete(id)) return;
|
||||
this.emit();
|
||||
}
|
||||
|
||||
private emit() {
|
||||
const snapshot = this.snapshot();
|
||||
for (const listener of this.listeners) listener(snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
export const browserPreviewActivity = new BrowserPreviewActivity();
|
||||
38
apps/web/src/lib/browser-url.test.ts
Normal file
38
apps/web/src/lib/browser-url.test.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { browserFrameSandbox, normalizeBrowserUrl } from "./browser-url";
|
||||
|
||||
describe("normalizeBrowserUrl", () => {
|
||||
it("defaults local dev servers to http and the web to https", () => {
|
||||
expect(normalizeBrowserUrl("localhost:5173")).toBe("http://localhost:5173/");
|
||||
expect(normalizeBrowserUrl("127.0.0.1:3000/app?x=1")).toBe("http://127.0.0.1:3000/app?x=1");
|
||||
expect(normalizeBrowserUrl("example.com/docs")).toBe("https://example.com/docs");
|
||||
expect(normalizeBrowserUrl(" https://example.com ")).toBe("https://example.com/");
|
||||
expect(normalizeBrowserUrl("http://example.com")).toBe("http://example.com/");
|
||||
});
|
||||
|
||||
it("rejects input that is not an http(s) address", () => {
|
||||
for (const input of [
|
||||
"",
|
||||
"hello world",
|
||||
"hello",
|
||||
"javascript:alert(1)",
|
||||
"file:///etc/passwd",
|
||||
"data:text/html,hi",
|
||||
"ftp://example.com",
|
||||
"https://user:fake-password@example.com",
|
||||
]) {
|
||||
expect(normalizeBrowserUrl(input), input).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("browserFrameSandbox", () => {
|
||||
it("never grants same-origin access to the app's own origin", () => {
|
||||
expect(browserFrameSandbox("https://app.test/x", "https://app.test")).not.toContain(
|
||||
"allow-same-origin",
|
||||
);
|
||||
expect(browserFrameSandbox("http://localhost:5173/", "https://app.test")).toContain(
|
||||
"allow-same-origin",
|
||||
);
|
||||
});
|
||||
});
|
||||
39
apps/web/src/lib/browser-url.ts
Normal file
39
apps/web/src/lib/browser-url.ts
Normal file
@ -0,0 +1,39 @@
|
||||
const LOCAL_HOST = /^(localhost|127(?:\.\d{1,3}){3}|\[::1\]|0\.0\.0\.0|[\w-]+\.local(?:host)?)$/i;
|
||||
|
||||
/**
|
||||
* Turns what someone types in the address bar into an http(s) URL. Local dev servers default to
|
||||
* http, everything else to https. Returns null for input that is not an address.
|
||||
*/
|
||||
export function normalizeBrowserUrl(input: string): string | null {
|
||||
const value = input.trim();
|
||||
if (!value || /\s/.test(value)) return null;
|
||||
const hasScheme = /^[a-z][a-z\d+.-]*:\/\//i.test(value);
|
||||
if (!hasScheme && /^[a-z][a-z\d+.-]*:(?!\d)/i.test(value)) return null;
|
||||
let candidate = value;
|
||||
if (!hasScheme) {
|
||||
const host = value.split(/[/?#]/, 1)[0]!.replace(/:\d+$/, "");
|
||||
if (!LOCAL_HOST.test(host) && !host.includes(".") && !/^\d+$/.test(host)) return null;
|
||||
candidate = `${LOCAL_HOST.test(host) ? "http" : "https"}://${value}`;
|
||||
}
|
||||
try {
|
||||
const url = new URL(candidate);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||
if (url.username || url.password || !url.hostname) return null;
|
||||
return url.toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sandbox for the web fallback. A page on the app's own origin must not get same-origin
|
||||
* access, or it could reach into the app around the sandbox.
|
||||
*/
|
||||
export function browserFrameSandbox(url: string, appOrigin: string): string {
|
||||
const base = "allow-scripts allow-forms allow-popups allow-modals allow-downloads";
|
||||
try {
|
||||
return new URL(url).origin === appOrigin ? base : `${base} allow-same-origin`;
|
||||
} catch {
|
||||
return base;
|
||||
}
|
||||
}
|
||||
31
apps/web/src/lib/csv-preview.test.ts
Normal file
31
apps/web/src/lib/csv-preview.test.ts
Normal file
@ -0,0 +1,31 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseCsvPreview } from "./csv-preview";
|
||||
|
||||
describe("parseCsvPreview", () => {
|
||||
it("reads quoted fields, doubled quotes and CRLF rows", () => {
|
||||
expect(parseCsvPreview('name,note\r\n"Smith, Ann","said ""hi"""\r\nBo,\n')).toEqual({
|
||||
rows: [
|
||||
["name", "note"],
|
||||
["Smith, Ann", 'said "hi"'],
|
||||
["Bo", ""],
|
||||
],
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a final row without a trailing newline and skips blank lines", () => {
|
||||
expect(parseCsvPreview("a,b\n\n1,2").rows).toEqual([
|
||||
["a", "b"],
|
||||
["1", "2"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps newlines inside quoted fields", () => {
|
||||
expect(parseCsvPreview('a\n"line 1\nline 2"').rows).toEqual([["a"], ["line 1\nline 2"]]);
|
||||
});
|
||||
|
||||
it("stops after the row limit", () => {
|
||||
const result = parseCsvPreview("1\n2\n3\n4\n", 2);
|
||||
expect(result).toEqual({ rows: [["1"], ["2"]], truncated: true });
|
||||
});
|
||||
});
|
||||
43
apps/web/src/lib/csv-preview.ts
Normal file
43
apps/web/src/lib/csv-preview.ts
Normal file
@ -0,0 +1,43 @@
|
||||
/** Parses enough of RFC 4180 to show a CSV as a table: quoted fields, doubled quotes, CRLF. */
|
||||
export function parseCsvPreview(
|
||||
text: string,
|
||||
maxRows = 500,
|
||||
): { rows: string[][]; truncated: boolean } {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = "";
|
||||
let quoted = false;
|
||||
let index = 0;
|
||||
const endRow = () => {
|
||||
row.push(field);
|
||||
field = "";
|
||||
if (row.length > 1 || row[0] !== "") rows.push(row);
|
||||
row = [];
|
||||
};
|
||||
while (index < text.length) {
|
||||
const char = text[index];
|
||||
if (quoted) {
|
||||
if (char === '"' && text[index + 1] === '"') {
|
||||
field += '"';
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
if (char === '"') quoted = false;
|
||||
else field += char;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (char === '"' && field === "") quoted = true;
|
||||
else if (char === ",") {
|
||||
row.push(field);
|
||||
field = "";
|
||||
} else if (char === "\n" || char === "\r") {
|
||||
endRow();
|
||||
if (char === "\r" && text[index + 1] === "\n") index += 1;
|
||||
if (rows.length > maxRows) break;
|
||||
} else field += char;
|
||||
index += 1;
|
||||
}
|
||||
if (rows.length <= maxRows && (field !== "" || row.length > 0)) endRow();
|
||||
return { rows: rows.slice(0, maxRows), truncated: rows.length > maxRows };
|
||||
}
|
||||
@ -925,6 +925,24 @@ describe("thread event reduction", () => {
|
||||
expect(next?.messages).toEqual(initial.messages);
|
||||
});
|
||||
|
||||
it("advances past a bot pointer announcement without touching messages", () => {
|
||||
const initial = snapshot([message("m1", [{ kind: "text", text: "hi" }], 3)], 3);
|
||||
const pointer = event({
|
||||
type: "thread.computer",
|
||||
seq: 4,
|
||||
runId: "run-1",
|
||||
payload: {
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [{ action: "click", x: 10, y: 20 }],
|
||||
},
|
||||
});
|
||||
expect(isThreadSnapshotEvent(pointer)).toBe(true);
|
||||
const next = reduceThreadSnapshot(initial, pointer);
|
||||
expect(next?.cursor).toBe(4);
|
||||
expect(next?.messages).toEqual(initial.messages);
|
||||
});
|
||||
|
||||
it("holds a tool call that lands mid-sentence until the sentence completes", () => {
|
||||
const initial = snapshot([]);
|
||||
|
||||
|
||||
@ -249,6 +249,7 @@ export function isThreadSnapshotEvent(event: ProductEvent): boolean {
|
||||
event.type === "thread.cloud_agent" ||
|
||||
event.type === "agent.tool.called" ||
|
||||
event.type === "agent.tool.completed" ||
|
||||
event.type === "thread.computer" ||
|
||||
event.type === "thread.message.created" ||
|
||||
event.type === "thread.message.updated" ||
|
||||
event.type === "thread.message.reaction" ||
|
||||
@ -439,7 +440,7 @@ export function reduceThreadSnapshot(
|
||||
};
|
||||
return { ...prev, cursor: event.seq, messages: [...remaining, next] };
|
||||
}
|
||||
if (event.type === "agent.tool.completed") {
|
||||
if (event.type === "agent.tool.completed" || event.type === "thread.computer") {
|
||||
return { ...prev, cursor: event.seq };
|
||||
}
|
||||
if (event.type === "thread.subagent") {
|
||||
|
||||
61
apps/web/src/lib/workspace-pane-pref.test.ts
Normal file
61
apps/web/src/lib/workspace-pane-pref.test.ts
Normal file
@ -0,0 +1,61 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
readPaneWidth,
|
||||
readScreenPaneHabit,
|
||||
writePaneWidth,
|
||||
writeScreenPaneHabit,
|
||||
} from "./workspace-pane-pref";
|
||||
|
||||
function stubStorage() {
|
||||
const store = new Map<string, string>();
|
||||
vi.stubGlobal("window", {
|
||||
localStorage: {
|
||||
getItem: (key: string) => store.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => {
|
||||
store.set(key, value);
|
||||
},
|
||||
removeItem: (key: string) => {
|
||||
store.delete(key);
|
||||
},
|
||||
},
|
||||
});
|
||||
return store;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("workspace pane preferences", () => {
|
||||
it("keeps the learned screen habit per user and bot", () => {
|
||||
const store = stubStorage();
|
||||
expect(readScreenPaneHabit("user-1", "bot-1")).toBe(0);
|
||||
writeScreenPaneHabit("user-1", "bot-1", 2);
|
||||
expect(readScreenPaneHabit("user-1", "bot-1")).toBe(2);
|
||||
expect(readScreenPaneHabit("user-1", "bot-2")).toBe(0);
|
||||
writeScreenPaneHabit("user-1", "bot-1", 0);
|
||||
expect(store.size).toBe(0);
|
||||
writeScreenPaneHabit(null, "bot-1", 3);
|
||||
expect(store.size).toBe(0);
|
||||
});
|
||||
|
||||
it("restores a stored pane width within its bounds", () => {
|
||||
stubStorage();
|
||||
expect(readPaneWidth("width", 500, 320, 900)).toBe(500);
|
||||
writePaneWidth("width", 640.4);
|
||||
expect(readPaneWidth("width", 500, 320, 900)).toBe(640);
|
||||
writePaneWidth("width", 2_000);
|
||||
expect(readPaneWidth("width", 500, 320, 900)).toBe(900);
|
||||
});
|
||||
|
||||
it("falls back when storage is unavailable", () => {
|
||||
vi.stubGlobal("window", {
|
||||
get localStorage(): Storage {
|
||||
throw new Error("blocked");
|
||||
},
|
||||
});
|
||||
expect(readScreenPaneHabit("user-1", "bot-1")).toBe(0);
|
||||
expect(readPaneWidth("width", 500, 320, 900)).toBe(500);
|
||||
expect(() => writePaneWidth("width", 600)).not.toThrow();
|
||||
});
|
||||
});
|
||||
52
apps/web/src/lib/workspace-pane-pref.ts
Normal file
52
apps/web/src/lib/workspace-pane-pref.ts
Normal file
@ -0,0 +1,52 @@
|
||||
import { parseScreenPaneHabit, type ScreenPaneHabit } from "@rakazo/core";
|
||||
|
||||
function screenHabitKey(userId: string | null | undefined, botId: string | null | undefined) {
|
||||
if (!userId || !botId) return null;
|
||||
return `rakazo:screen-pane:${userId}:${botId}`;
|
||||
}
|
||||
|
||||
export function readScreenPaneHabit(
|
||||
userId: string | null | undefined,
|
||||
botId: string | null | undefined,
|
||||
): ScreenPaneHabit {
|
||||
const key = screenHabitKey(userId, botId);
|
||||
if (!key) return 0;
|
||||
try {
|
||||
return parseScreenPaneHabit(window.localStorage.getItem(key));
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
export function writeScreenPaneHabit(
|
||||
userId: string | null | undefined,
|
||||
botId: string | null | undefined,
|
||||
habit: ScreenPaneHabit,
|
||||
): void {
|
||||
const key = screenHabitKey(userId, botId);
|
||||
if (!key) return;
|
||||
try {
|
||||
if (habit === 0) window.localStorage.removeItem(key);
|
||||
else window.localStorage.setItem(key, String(habit));
|
||||
} catch {
|
||||
// Private mode or a full quota only loses the learned default.
|
||||
}
|
||||
}
|
||||
|
||||
export function readPaneWidth(storageKey: string, fallback: number, min: number, max: number) {
|
||||
try {
|
||||
const stored = Number(window.localStorage.getItem(storageKey));
|
||||
if (Number.isFinite(stored) && stored > 0) return Math.max(min, Math.min(max, stored));
|
||||
} catch {
|
||||
// Fall through to the default width.
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function writePaneWidth(storageKey: string, width: number): void {
|
||||
try {
|
||||
window.localStorage.setItem(storageKey, String(Math.round(width)));
|
||||
} catch {
|
||||
// The width still applies for this session.
|
||||
}
|
||||
}
|
||||
@ -1,20 +1,31 @@
|
||||
import { Trans } from "@lingui/react/macro";
|
||||
import { Button } from "@rakazo/ui-web";
|
||||
import { useEffect, useState } from "react";
|
||||
import { IntegrationSetup } from "../components/integrations/IntegrationSetup";
|
||||
import { Suspense, useEffect, useState } from "react";
|
||||
import { desktopBridge } from "../lib/desktop";
|
||||
import { lazyWithRecovery } from "../lib/lazy-recovery";
|
||||
import { rpc } from "../lib/rpc";
|
||||
import { ModelSettingsOverlay } from "./ModelSettingsOverlay";
|
||||
|
||||
const IntegrationSetup = lazyWithRecovery("integrations/IntegrationSetup", () =>
|
||||
import("../components/integrations/IntegrationSetup").then((module) => ({
|
||||
default: module.IntegrationSetup,
|
||||
})),
|
||||
);
|
||||
const ModelSettingsOverlay = lazyWithRecovery("pages/ModelSettingsOverlay", () =>
|
||||
import("./ModelSettingsOverlay").then((module) => ({ default: module.ModelSettingsOverlay })),
|
||||
);
|
||||
|
||||
export function LocalSettingsPage() {
|
||||
const [section, setSection] = useState<"models" | "integrations" | null>(null);
|
||||
const hasLocalBridge = Boolean(desktopBridge()?.localSettings);
|
||||
const [ready, setReady] = useState(false);
|
||||
const [error, setError] = useState(false);
|
||||
const [error, setError] = useState(!hasLocalBridge);
|
||||
useEffect(() => {
|
||||
if (!hasLocalBridge) return;
|
||||
void rpc.integrationSetup
|
||||
.get()
|
||||
.then(() => setReady(true))
|
||||
.catch(() => setError(true));
|
||||
}, []);
|
||||
}, [hasLocalBridge]);
|
||||
return (
|
||||
<main className="h-full overflow-auto bg-background px-6 py-12">
|
||||
<div className="mx-auto max-w-xl space-y-6">
|
||||
@ -43,9 +54,15 @@ export function LocalSettingsPage() {
|
||||
</Button>
|
||||
</nav>
|
||||
{section === "models" ? (
|
||||
<ModelSettingsOverlay onClose={() => setSection(null)} localOwner />
|
||||
<Suspense fallback={null}>
|
||||
<ModelSettingsOverlay onClose={() => setSection(null)} localOwner />
|
||||
</Suspense>
|
||||
) : null}
|
||||
{section === "integrations" ? (
|
||||
<Suspense fallback={null}>
|
||||
<IntegrationSetup serverSetup managedOnly />
|
||||
</Suspense>
|
||||
) : null}
|
||||
{section === "integrations" ? <IntegrationSetup serverSetup managedOnly /> : null}
|
||||
</>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
@ -49,11 +49,13 @@ import {
|
||||
normalizeCreateBotProfile,
|
||||
} from "@rakazo/contracts";
|
||||
import {
|
||||
applyWorkspaceFocus,
|
||||
attachmentsForThread,
|
||||
buildComposerMentionOptions,
|
||||
type ComposerMention,
|
||||
clampMentionHighlightIndex,
|
||||
cronFromPreset,
|
||||
deriveWorkspaceFocus,
|
||||
groupBotsForSidebar,
|
||||
inferAttachmentMimeType,
|
||||
isActive,
|
||||
@ -61,21 +63,27 @@ import {
|
||||
isRunTerminalEvent,
|
||||
isToolActivityBlock,
|
||||
latestAnswerableAskMessageId,
|
||||
liveScreenRunId,
|
||||
matchVoiceByName,
|
||||
mentionChipKey,
|
||||
type PaneState,
|
||||
parseVoiceChatCommand,
|
||||
projectMessageReactions,
|
||||
recordScreenPaneChoice,
|
||||
reorderBotTo,
|
||||
resolveComposerSendPlan,
|
||||
resolveMentionPickerKey,
|
||||
runThreadSubscription,
|
||||
SLASH_ACTIONS,
|
||||
type SlashActionId,
|
||||
screenPaneMode,
|
||||
searchHitThreadTarget,
|
||||
serializeComposerPrompt,
|
||||
speechFromBlocks,
|
||||
truncateSlashDescription,
|
||||
userVisibleMessages,
|
||||
type WorkspaceArtifact,
|
||||
type WorkspaceFocus,
|
||||
} from "@rakazo/core";
|
||||
import {
|
||||
AvatarStyleProvider,
|
||||
@ -110,6 +118,7 @@ import {
|
||||
Copy,
|
||||
Folder,
|
||||
Gauge,
|
||||
Globe,
|
||||
GripVertical,
|
||||
LayoutGrid,
|
||||
Lock,
|
||||
@ -155,10 +164,12 @@ import {
|
||||
import { createPortal } from "react-dom";
|
||||
import { useNavigate, useParams, useSearchParams } from "react-router-dom";
|
||||
import { ArtifactFileCard } from "../components/ArtifactFileCard";
|
||||
import { ArtifactPane } from "../components/ArtifactPane";
|
||||
import { AskCard } from "../components/AskCard";
|
||||
import { ActiveBotGlyph, CollaborationMarker } from "../components/ai/CollaborationMarker";
|
||||
import { DelegatedTaskCard, SentMessageCard } from "../components/ai/DelegatedTaskCard";
|
||||
import { ProjectCreatedCard } from "../components/ai/ProjectCreatedCard";
|
||||
import { InAppBrowser } from "../components/browser/InAppBrowser";
|
||||
import { CloudAgentCard } from "../components/CloudAgentCard";
|
||||
import { ComputerMaintenanceActions } from "../components/ComputerMaintenanceActions";
|
||||
import {
|
||||
@ -166,6 +177,7 @@ import {
|
||||
computersAreUnavailable,
|
||||
} from "../components/ComputersUnavailableHint";
|
||||
import { ComputerUpdateProgress } from "../components/ComputerUpdateProgress";
|
||||
import { AgentCursor } from "../components/computer/AgentCursor";
|
||||
import { DataTrustPanel } from "../components/DataTrustPanel";
|
||||
import { MainInbox } from "../components/MainInbox";
|
||||
import { MessageHoverMetadata } from "../components/MessageHoverMetadata";
|
||||
@ -177,8 +189,10 @@ import { TeachComputerOverlayControl } from "../components/teach/TeachComputerOv
|
||||
import { TeachRecordingChrome, TeachStopButton } from "../components/teach/TeachRecordingChrome";
|
||||
import { VoiceAuroraRim } from "../components/voice/VoiceAurora";
|
||||
import { useSidebarResize } from "../hooks/useSidebarResize";
|
||||
import { useWideLayout } from "../hooks/useWideLayout";
|
||||
import { readActivityMode, writeActivityMode } from "../lib/activity-mode";
|
||||
import { needsYouChipLabel, rosterActivitySubtitle, rosterStatusDot } from "../lib/agent-activity";
|
||||
import { agentPointerFeed } from "../lib/agent-pointer";
|
||||
import type { ArtifactTarget } from "../lib/artifact-open";
|
||||
import { authClient } from "../lib/auth";
|
||||
import { takeInitialBootstrap } from "../lib/bootstrap";
|
||||
@ -192,8 +206,6 @@ import {
|
||||
requestBrowserNotificationPermission,
|
||||
shouldNotifyBrowser,
|
||||
} from "../lib/browser-notifications";
|
||||
import type { BrowserPreviewCursor } from "../lib/browser-preview-context";
|
||||
import { browserPreviewActivity } from "../lib/browser-preview-context";
|
||||
import { loadComputerScreen } from "../lib/computer-screen";
|
||||
import { desktopBridge } from "../lib/desktop";
|
||||
import { scheduleFocusPrompt } from "../lib/focus-prompt";
|
||||
@ -247,6 +259,7 @@ import {
|
||||
transcriptMovedDown,
|
||||
} from "../lib/transcript-scroll";
|
||||
import { speaker } from "../lib/tts";
|
||||
import { readScreenPaneHabit, writeScreenPaneHabit } from "../lib/workspace-pane-pref";
|
||||
import { ActivityList } from "./ActivityList";
|
||||
import type { ContextMenuPosition } from "./BotContextMenu";
|
||||
import { CreateGroupForm, GroupSettings, memberName } from "./GroupPanel";
|
||||
@ -325,6 +338,7 @@ type Panel =
|
||||
| "create-group"
|
||||
| "group-settings"
|
||||
| "browser"
|
||||
| "artifact"
|
||||
| null;
|
||||
|
||||
type PendingAttachment = {
|
||||
@ -349,6 +363,14 @@ const THREAD_SNAPSHOT_TIMEOUT_MS = 2_000;
|
||||
const VOICE_STATUS_REFRESH_TIMEOUT_MS = 10_000;
|
||||
const MOBILE_SIDEBAR_SWIPE_EDGE_PX = 32;
|
||||
const MOBILE_SIDEBAR_SWIPE_DISTANCE_PX = 56;
|
||||
/** The screen lingers after the work so its final state and the bot's reply land first. */
|
||||
const AUTO_SCREEN_CLOSE_DELAY_MS = 4_000;
|
||||
/** How long one pointer announcement counts as the bot working on its screen. */
|
||||
const SCREEN_ACTIVE_MS = 5_000;
|
||||
|
||||
function defaultWorkspacePaneWidth(): number {
|
||||
return Math.round(Math.max(420, Math.min(760, window.innerWidth * 0.42)));
|
||||
}
|
||||
|
||||
function threadSnapshotSignal(parent: AbortSignal): AbortSignal {
|
||||
return AbortSignal.any([parent, AbortSignal.timeout(THREAD_SNAPSHOT_TIMEOUT_MS)]);
|
||||
@ -381,31 +403,6 @@ function readCollapsedSidebarSections(userId: string | null | undefined): Set<st
|
||||
}
|
||||
}
|
||||
|
||||
function copyBrowserPreviewActivities(
|
||||
activities: ReadonlyMap<string, BrowserPreviewCursor>,
|
||||
): Map<string, BrowserPreviewCursor> {
|
||||
return new Map(activities);
|
||||
}
|
||||
|
||||
function sameBrowserPreviewActivities(
|
||||
current: ReadonlyMap<string, BrowserPreviewCursor>,
|
||||
next: ReadonlyMap<string, BrowserPreviewCursor>,
|
||||
): boolean {
|
||||
if (current.size !== next.size) return false;
|
||||
for (const [id, cursor] of next) {
|
||||
const existing = current.get(id);
|
||||
if (
|
||||
!existing ||
|
||||
existing.x !== cursor.x ||
|
||||
existing.y !== cursor.y ||
|
||||
existing.label !== cursor.label
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export type NavPillar = "heute" | "team" | "projekte";
|
||||
|
||||
function stripSpaceTitlePrefix(title: string | null, spaceName: string): string | null {
|
||||
@ -461,12 +458,24 @@ export function ShellPage() {
|
||||
width: leftSidebarWidth,
|
||||
isResizing: isResizingLeft,
|
||||
startResizing: startResizingLeft,
|
||||
} = useSidebarResize(316, "left", 240, 450);
|
||||
} = useSidebarResize(316, "left", 240, 450, "rakazo:left-sidebar-width");
|
||||
const {
|
||||
width: rightSidebarWidth,
|
||||
isResizing: isResizingRight,
|
||||
startResizing: startResizingRight,
|
||||
} = useSidebarResize(384, "right", 280, 500);
|
||||
} = useSidebarResize(384, "right", 280, 500, "rakazo:side-panel-width");
|
||||
// The screen and presented files get a wider pane than settings: they are what the user watches.
|
||||
const {
|
||||
width: workspacePaneWidth,
|
||||
isResizing: isResizingWorkspacePane,
|
||||
startResizing: startResizingWorkspacePane,
|
||||
} = useSidebarResize(
|
||||
defaultWorkspacePaneWidth(),
|
||||
"right",
|
||||
360,
|
||||
960,
|
||||
"rakazo:workspace-pane-width",
|
||||
);
|
||||
|
||||
const [heuteActionCount, setHeuteActionCount] = useState(0);
|
||||
const [archivedOpen, setArchivedOpen] = useState(false);
|
||||
@ -480,16 +489,16 @@ export function ShellPage() {
|
||||
setBotsSidebarCollapsed(readBotsSidebarCollapsed(userId));
|
||||
}, [userId]);
|
||||
|
||||
// ── Browser Preview ──────────────────────────────────────────────
|
||||
const bridge = desktopBridge();
|
||||
const [isDesktop] = useState(Boolean(bridge?.browserPreview));
|
||||
const [panel, setPanel] = useState<Panel>(null);
|
||||
const [browserUrl, setBrowserUrl] = useState("http://localhost:5173");
|
||||
const [devServerRunning, setDevServerRunning] = useState(false);
|
||||
const [browserLoading, setBrowserLoading] = useState(false);
|
||||
const [agentActivities, setAgentActivities] = useState<
|
||||
Map<string, { x: number; y: number; label: string }>
|
||||
>(new Map());
|
||||
/** The pane the conversation's focus opened; cleared as soon as the user changes the pane. */
|
||||
const [autoPanel, setAutoPanel] = useState<Panel>(null);
|
||||
const [artifactPane, setArtifactPane] = useState<WorkspaceArtifact | null>(null);
|
||||
/** Run that last pointed at the bot's screen; the live screen only follows real screen work. */
|
||||
const [screenRunId, setScreenRunId] = useState<string | null>(null);
|
||||
const [screenActive, setScreenActive] = useState(false);
|
||||
const screenActiveTimer = useRef<number | undefined>(undefined);
|
||||
const [screenHabit, setScreenHabit] = useState(0);
|
||||
const wideLayout = useWideLayout();
|
||||
|
||||
const [showCreateBotModal, setShowCreateBotModal] = useState(false);
|
||||
|
||||
@ -505,60 +514,6 @@ export function ShellPage() {
|
||||
useSensor(TouchSensor, { activationConstraint: { distance: 8 } }),
|
||||
);
|
||||
|
||||
// Browser Preview lifecycle
|
||||
useEffect(() => {
|
||||
const bp = bridge?.browserPreview;
|
||||
if (!bp) return;
|
||||
const unsubNav = bp.onNavigate((url) => setBrowserUrl(url));
|
||||
const unsubLoading = bp.onLoadingProgress((p) => setBrowserLoading(p < 1));
|
||||
bp.getState()
|
||||
.then((s) => {
|
||||
setBrowserUrl(s.url);
|
||||
setDevServerRunning(s.devServerRunning);
|
||||
})
|
||||
.catch(() => {});
|
||||
return () => {
|
||||
unsubNav();
|
||||
unsubLoading();
|
||||
};
|
||||
}, [bridge]);
|
||||
|
||||
const startDevServer = useCallback(async () => {
|
||||
try {
|
||||
await bridge?.browserPreview?.devServer.start();
|
||||
setDevServerRunning(true);
|
||||
} catch {
|
||||
/* */
|
||||
}
|
||||
}, [bridge]);
|
||||
const stopDevServer = useCallback(async () => {
|
||||
try {
|
||||
await bridge?.browserPreview?.devServer.stop();
|
||||
setDevServerRunning(false);
|
||||
} catch {
|
||||
/* */
|
||||
}
|
||||
}, [bridge]);
|
||||
const navigateBrowser = useCallback(
|
||||
(url: string) => {
|
||||
bridge?.browserPreview?.navigate(url).catch(() => {});
|
||||
},
|
||||
[bridge],
|
||||
);
|
||||
const reloadBrowser = useCallback(() => {
|
||||
bridge?.browserPreview?.reload().catch(() => {});
|
||||
}, [bridge]);
|
||||
|
||||
// Agent activity → browser preview cursors
|
||||
useEffect(() => {
|
||||
const unsub = browserPreviewActivity.subscribe((acts) => {
|
||||
setAgentActivities((current) =>
|
||||
sameBrowserPreviewActivities(current, acts) ? current : copyBrowserPreviewActivities(acts),
|
||||
);
|
||||
});
|
||||
return unsub;
|
||||
}, []);
|
||||
|
||||
const [query, setQuery] = useState("");
|
||||
const [searchHits, setSearchHits] = useState<SearchHit[]>([]);
|
||||
const [searchLoading, setSearchLoading] = useState(false);
|
||||
@ -1542,6 +1497,18 @@ export function ShellPage() {
|
||||
void refreshThread(active.id).catch(() => undefined);
|
||||
} else if (isComputerStatusEvent(event)) {
|
||||
void refreshComputerScreen(active.id).catch(() => undefined);
|
||||
} else if (event.type === "thread.computer") {
|
||||
agentPointerFeed.publish(event.botId, event.payload, event.createdAt);
|
||||
const screenRun = liveScreenRunId(event);
|
||||
if (screenRun) {
|
||||
setScreenRunId(screenRun);
|
||||
setScreenActive(true);
|
||||
window.clearTimeout(screenActiveTimer.current);
|
||||
screenActiveTimer.current = window.setTimeout(
|
||||
() => setScreenActive(false),
|
||||
SCREEN_ACTIVE_MS,
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
@ -2186,6 +2153,83 @@ export function ShellPage() {
|
||||
() => (inGroup ? { groupId: groupId ?? "" } : { botId: active?.id ?? "" }),
|
||||
[active?.id, groupId, inGroup],
|
||||
);
|
||||
const workspaceFocus = useMemo(
|
||||
() =>
|
||||
deriveWorkspaceFocus(
|
||||
activeSnapshot ? { ...activeSnapshot, computer: inGroup ? null : computer } : null,
|
||||
{ screenRunId, screenActive, screenMode: screenPaneMode(screenHabit) },
|
||||
),
|
||||
[activeSnapshot, computer, inGroup, screenActive, screenHabit, screenRunId],
|
||||
);
|
||||
const workspaceFocusRef = useRef<{ threadId: string; focus: WorkspaceFocus } | null>(null);
|
||||
const autoPanelRef = useRef(autoPanel);
|
||||
autoPanelRef.current = autoPanel;
|
||||
const autoCloseTimer = useRef<number | undefined>(undefined);
|
||||
useEffect(() => () => window.clearTimeout(autoCloseTimer.current), []);
|
||||
// The pane follows the bot's work on focus edges only, so it never jumps while nothing changed.
|
||||
useEffect(() => {
|
||||
const threadId = activeSnapshot?.threadId;
|
||||
const sameThread = Boolean(threadId) && workspaceFocusRef.current?.threadId === threadId;
|
||||
// New work or another conversation cancels a pending close; an unrelated edge does not.
|
||||
if (!sameThread || workspaceFocus.pane !== null) window.clearTimeout(autoCloseTimer.current);
|
||||
if (!threadId) {
|
||||
workspaceFocusRef.current = null;
|
||||
return;
|
||||
}
|
||||
const previous = sameThread ? (workspaceFocusRef.current?.focus ?? null) : null;
|
||||
workspaceFocusRef.current = { threadId, focus: workspaceFocus };
|
||||
// Narrow windows keep the conversation full width; a pane would cover it.
|
||||
if (!wideLayout) return;
|
||||
const paneState: PaneState = {
|
||||
open:
|
||||
panel === "computer"
|
||||
? "screen"
|
||||
: panel === "artifact"
|
||||
? "artifact"
|
||||
: panel
|
||||
? "other"
|
||||
: null,
|
||||
auto: panel !== null && panel === autoPanel,
|
||||
};
|
||||
const next = applyWorkspaceFocus(previous, workspaceFocus, paneState);
|
||||
if (!next || next.open === "other") return;
|
||||
if (next.open === null) {
|
||||
const closing = autoPanelRef.current;
|
||||
autoCloseTimer.current = window.setTimeout(() => {
|
||||
if (closing === null || autoPanelRef.current !== closing) return;
|
||||
setPanel(null);
|
||||
setAutoPanel(null);
|
||||
}, AUTO_SCREEN_CLOSE_DELAY_MS);
|
||||
return;
|
||||
}
|
||||
if (next.open === "artifact") setArtifactPane(workspaceFocus.artifact);
|
||||
const nextPanel: Panel = next.open === "screen" ? "computer" : next.open;
|
||||
setPanel(nextPanel);
|
||||
setAutoPanel(next.auto ? nextPanel : null);
|
||||
}, [workspaceFocus.key, activeSnapshot?.threadId, wideLayout]);
|
||||
const learnScreenPane = useCallback(
|
||||
(choice: "opened" | "dismissed") => {
|
||||
const botId = activeBotId.current;
|
||||
if (!botId) return;
|
||||
const next = recordScreenPaneChoice(screenHabit, choice);
|
||||
setScreenHabit(next);
|
||||
writeScreenPaneHabit(userId, botId, next);
|
||||
},
|
||||
[screenHabit, userId],
|
||||
);
|
||||
useEffect(() => {
|
||||
if (autoPanel === null || panel === autoPanel) return;
|
||||
// Closing the live screen while the bot still works on it teaches this bot's default.
|
||||
const focus = workspaceFocusRef.current?.focus;
|
||||
if (autoPanel === "computer" && panel === null && focus?.pane === "screen" && !focus.takeover) {
|
||||
learnScreenPane("dismissed");
|
||||
}
|
||||
setAutoPanel(null);
|
||||
}, [panel]);
|
||||
const openArtifactBeside = useCallback((artifact: WorkspaceArtifact) => {
|
||||
setArtifactPane(artifact);
|
||||
setPanel("artifact");
|
||||
}, []);
|
||||
const transcriptMembers = activeSnapshot?.members ?? activeGroup?.members;
|
||||
// Bots of every space, so a group message from a bot that is not a member still has a speaker.
|
||||
const knownBots = useMemo(() => {
|
||||
@ -2237,20 +2281,6 @@ export function ShellPage() {
|
||||
}));
|
||||
}, [activeSnapshot, enrichStageMember, inGroup, resolveTranscriptBot, workingRuns]);
|
||||
|
||||
// Working bots → browser preview activity cursors
|
||||
useEffect(() => {
|
||||
if (workingBots.length === 0) return;
|
||||
for (const bot of workingBots) {
|
||||
if (!bot.botId) continue;
|
||||
const label = bot.name ?? bot.botId.slice(0, 8);
|
||||
browserPreviewActivity.set(bot.botId, 50, 40, label);
|
||||
}
|
||||
const activeBotIds = new Set(workingBots.map((b) => b.botId).filter(Boolean));
|
||||
for (const id of browserPreviewActivity.snapshot().keys()) {
|
||||
if (!activeBotIds.has(id)) browserPreviewActivity.clear(id);
|
||||
}
|
||||
}, [workingBots]);
|
||||
|
||||
const encounterMode = Boolean((active || activeGroup) && !showProjectTimeline && !callOpen);
|
||||
const voiceEncounterActive = encounterMode || callOpen || voiceEntry;
|
||||
const voiceCallBot = useMemo(() => {
|
||||
@ -3208,8 +3238,22 @@ export function ShellPage() {
|
||||
setComputerOpen(false);
|
||||
setComputerError(null);
|
||||
setComputerErrorFromScreen(false);
|
||||
setScreenRunId(null);
|
||||
setScreenActive(false);
|
||||
window.clearTimeout(screenActiveTimer.current);
|
||||
}, [active?.id]);
|
||||
|
||||
useEffect(() => {
|
||||
setScreenHabit(readScreenPaneHabit(userId, active?.id));
|
||||
}, [active?.id, userId]);
|
||||
|
||||
// A presented file belongs to its conversation, and a new conversation starts without auto panes.
|
||||
useEffect(() => {
|
||||
setPanel((current) => (current === "artifact" ? null : current));
|
||||
setArtifactPane(null);
|
||||
setAutoPanel(null);
|
||||
}, [active?.id, groupId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (focusPromptBotIdRef.current && focusPromptBotIdRef.current !== active?.id) {
|
||||
cancelFocusPrompt();
|
||||
@ -3326,6 +3370,9 @@ export function ShellPage() {
|
||||
}
|
||||
|
||||
const embeddedScreenUrl = embeddableScreenUrl(screenUrl);
|
||||
// Only what the work opened gets the wide pane; settings, routines and a hand-opened screen keep
|
||||
// one width so switching between them never jumps.
|
||||
const workspacePaneOpen = panel === "artifact" || (panel === "computer" && autoPanel === panel);
|
||||
const hasControl = userHoldsComputerControl(computer, active?.id);
|
||||
const hideScreenLoadError = computerErrorFromScreen && Boolean(embeddedScreenUrl);
|
||||
const computerScreenError =
|
||||
@ -4492,12 +4539,23 @@ export function ShellPage() {
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
title={t`Browser`}
|
||||
aria-label={t`Browser`}
|
||||
onClick={() => setPanel(panel === "browser" ? null : "browser")}
|
||||
data-active={panel === "browser" ? "" : undefined}
|
||||
className="app-no-drag grid h-[30px] w-[34px] place-items-center rounded-[9px] hover:bg-accent data-active:bg-accent"
|
||||
>
|
||||
<Globe strokeWidth={1.7} className="text-foreground/75 size-icon-lg" />
|
||||
</button>
|
||||
{!inGroup && active ? (
|
||||
<button
|
||||
type="button"
|
||||
title={t`Agent computer`}
|
||||
onClick={() => {
|
||||
const next = panel === "computer" ? null : "computer";
|
||||
if (next === "computer" && transcriptRunning) learnScreenPane("opened");
|
||||
setPanel(next);
|
||||
if (next === "computer" && active) {
|
||||
// Refresh run/computer so Take control isn't stuck on a stale busyBotName.
|
||||
@ -4588,6 +4646,7 @@ export function ShellPage() {
|
||||
}}
|
||||
scrollRef={messageScroll}
|
||||
artifactTarget={transcriptArtifactTarget}
|
||||
onOpenArtifact={wideLayout ? openArtifactBeside : undefined}
|
||||
messages={transcriptMessages}
|
||||
olderCursor={activeSnapshot?.olderCursor ?? null}
|
||||
loadingOlder={loadingOlder}
|
||||
@ -4693,6 +4752,7 @@ export function ShellPage() {
|
||||
<aside
|
||||
data-testid="side-panel"
|
||||
data-panel={panel ?? "closed"}
|
||||
data-auto={panel !== null && panel === autoPanel ? "" : undefined}
|
||||
style={{
|
||||
width:
|
||||
panel &&
|
||||
@ -4701,7 +4761,9 @@ export function ShellPage() {
|
||||
panel === "create" ||
|
||||
panel === "create-group" ||
|
||||
panel === "browser")
|
||||
? rightSidebarWidth
|
||||
? workspacePaneOpen
|
||||
? workspacePaneWidth
|
||||
: rightSidebarWidth
|
||||
: 0,
|
||||
}}
|
||||
className={`absolute inset-y-0 end-0 z-20 flex min-h-0 shrink-0 flex-col overflow-hidden bg-background md:rounded-2xl md:border md:border-border md:shadow-2xl md:relative ${
|
||||
@ -4715,13 +4777,24 @@ export function ShellPage() {
|
||||
)
|
||||
? "max-w-[384px] md:max-w-none opacity-100"
|
||||
: "pointer-events-none w-0 opacity-0 md:opacity-0 md:w-0"
|
||||
} ${!isResizingRight ? "transition-[width,opacity] duration-300 ease-[cubic-bezier(0.16,1,0.3,1)]" : ""}`}
|
||||
} ${!(workspacePaneOpen ? isResizingWorkspacePane : isResizingRight) ? "transition-[width,opacity] duration-300 ease-[cubic-bezier(0.16,1,0.3,1)] motion-reduce:transition-none" : ""}`}
|
||||
>
|
||||
<div
|
||||
onPointerDown={startResizingRight}
|
||||
onPointerDown={workspacePaneOpen ? startResizingWorkspacePane : startResizingRight}
|
||||
className="absolute -left-2 top-0 bottom-0 w-4 cursor-col-resize z-50 hidden md:block"
|
||||
/>
|
||||
{panel && (active || activeGroup || panel === "create" || panel === "create-group") ? (
|
||||
{panel === "artifact" ? (
|
||||
artifactPane ? (
|
||||
<ArtifactPane
|
||||
key={artifactPane.artifactId}
|
||||
target={transcriptArtifactTarget}
|
||||
artifact={artifactPane}
|
||||
onClose={() => setPanel(null)}
|
||||
/>
|
||||
) : null
|
||||
) : panel === "browser" ? (
|
||||
<InAppBrowser onClose={() => setPanel(null)} />
|
||||
) : panel && (active || activeGroup || panel === "create" || panel === "create-group") ? (
|
||||
<div className="rk-scroll h-full w-full overflow-y-auto px-5 lg:px-7 2xl:px-9 py-[17px]">
|
||||
{panel !== "routine" &&
|
||||
panel !== "create" &&
|
||||
@ -4761,17 +4834,6 @@ export function ShellPage() {
|
||||
<Settings strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
) : null}
|
||||
{!panel && active ? (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Browser preview`}
|
||||
onClick={() => setPanel("browser")}
|
||||
className="text-muted-foreground"
|
||||
>
|
||||
<PanelRight strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
) : null}
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
@ -4783,80 +4845,6 @@ export function ShellPage() {
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
{panel === "browser" ? (
|
||||
<div className="flex h-full flex-col">
|
||||
<div className="flex items-center gap-1.5 border-b px-3 py-2">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
onClick={reloadBrowser}
|
||||
aria-label={t`Reload`}
|
||||
className="size-icon-sm text-muted-foreground"
|
||||
>
|
||||
<RefreshCw strokeWidth={1.7} className="size-4" />
|
||||
</Button>
|
||||
<input
|
||||
type="url"
|
||||
value={browserUrl}
|
||||
onChange={(e) => setBrowserUrl(e.target.value)}
|
||||
onBlur={() => navigateBrowser(browserUrl)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter") navigateBrowser(browserUrl);
|
||||
}}
|
||||
placeholder="http://localhost:5173"
|
||||
className="flex-1 rounded-md bg-muted/60 px-2.5 py-1 text-xs text-foreground placeholder:text-muted-foreground/60 outline-none focus:ring-1 focus:ring-ring/50"
|
||||
/>
|
||||
{isDesktop ? (
|
||||
devServerRunning ? (
|
||||
<span className="inline-flex items-center gap-1 rounded-full bg-muted px-2 py-0.5 text-ui-label font-medium text-foreground">
|
||||
<span className="size-1.5 rounded-full bg-success" />
|
||||
<Trans>Dev</Trans>
|
||||
</span>
|
||||
) : (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
onClick={startDevServer}
|
||||
className="text-ui-label text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<Trans>Start Dev</Trans>
|
||||
</Button>
|
||||
)
|
||||
) : null}
|
||||
{browserLoading && (
|
||||
<div className="h-0.5 w-full rounded-full bg-muted/30">
|
||||
<div className="h-full w-2/3 animate-pulse rounded-full bg-primary/60" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="relative flex-1 overflow-hidden bg-background">
|
||||
<div className="relative h-full w-full" data-testid="browser-preview">
|
||||
{agentActivities.size > 0 && (
|
||||
<div className="pointer-events-none absolute inset-0">
|
||||
{Array.from(agentActivities.entries()).map(([agentId, activity]) => {
|
||||
const cx = Math.min(100, Math.max(0, activity.x)) + "%";
|
||||
const cy = Math.min(100, Math.max(0, activity.y)) + "%";
|
||||
return (
|
||||
<div
|
||||
key={agentId}
|
||||
className="absolute transition-[left,top] duration-300 ease-out"
|
||||
style={{ left: cx, top: cy }}
|
||||
>
|
||||
<div className="relative">
|
||||
<div className="size-3 rounded-full bg-primary/60 ring-2 ring-primary" />
|
||||
<div className="absolute -right-1 -top-4 whitespace-nowrap rounded bg-overlay px-1.5 py-0.5 text-ui-label text-foreground shadow-sm">
|
||||
{activity.label}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
{panel === "computer" && active ? (
|
||||
<div>
|
||||
<div
|
||||
@ -4870,14 +4858,17 @@ export function ShellPage() {
|
||||
) : computer?.kind === "desktop" ? (
|
||||
<DesktopKindEmptyState className="grid h-full place-items-center px-6 text-center text-sm text-muted-foreground/80" />
|
||||
) : computer?.state === "running" && embeddedScreenUrl && !computerScreenError ? (
|
||||
<iframe
|
||||
title={t`Bot screen preview`}
|
||||
src={embeddedScreenUrl}
|
||||
sandbox={screenIframeSandbox(embeddedScreenUrl)}
|
||||
className="h-full w-full border-0 bg-black"
|
||||
allow="clipboard-read; clipboard-write"
|
||||
style={{ pointerEvents: "none" }}
|
||||
/>
|
||||
<>
|
||||
<iframe
|
||||
title={t`Bot screen preview`}
|
||||
src={embeddedScreenUrl}
|
||||
sandbox={screenIframeSandbox(embeddedScreenUrl)}
|
||||
className="h-full w-full border-0 bg-black"
|
||||
allow="clipboard-read; clipboard-write"
|
||||
style={{ pointerEvents: "none" }}
|
||||
/>
|
||||
<AgentCursor botId={active.id} color={active.color} name={active.name} />
|
||||
</>
|
||||
) : (
|
||||
<div className="grid h-full place-items-center px-6 text-center text-sm text-muted-foreground/80">
|
||||
{computerScreenError ??
|
||||
@ -5482,6 +5473,7 @@ export function ShellPage() {
|
||||
{newSpaceOpen ? (
|
||||
<NewSpaceDialog
|
||||
variant={newSpaceDialogVariant}
|
||||
canLinkFolder={bootstrapMe?.isDeploymentOwner === true}
|
||||
onCancel={() => setNewSpaceOpen(false)}
|
||||
onConfirm={async ({ name, workspacePath }) => {
|
||||
const space = await rpc.spaces.create({
|
||||
@ -5767,6 +5759,9 @@ export function ShellPage() {
|
||||
pointerEvents: recordingSkill || !hasControl ? "none" : "auto",
|
||||
}}
|
||||
/>
|
||||
{active && !hasControl ? (
|
||||
<AgentCursor botId={active.id} color={active.color} name={active.name} />
|
||||
) : null}
|
||||
{active ? (
|
||||
<TeachCaptureOverlay
|
||||
botId={active.id}
|
||||
@ -5798,6 +5793,7 @@ export function ShellPage() {
|
||||
const Transcript = memo(function Transcript({
|
||||
scrollRef,
|
||||
artifactTarget,
|
||||
onOpenArtifact,
|
||||
messages,
|
||||
olderCursor,
|
||||
loadingOlder,
|
||||
@ -5828,6 +5824,7 @@ const Transcript = memo(function Transcript({
|
||||
}: {
|
||||
scrollRef: RefObject<HTMLDivElement | null>;
|
||||
artifactTarget: ArtifactTarget;
|
||||
onOpenArtifact?: (artifact: WorkspaceArtifact) => void;
|
||||
messages: ThreadMessage[];
|
||||
olderCursor: number | null;
|
||||
loadingOlder: boolean;
|
||||
@ -5967,6 +5964,43 @@ const Transcript = memo(function Transcript({
|
||||
if (following.current) snapToEnd();
|
||||
}, [messages, running, snapToEnd]);
|
||||
|
||||
// A question the bot waits on can scroll out of view; offer a way back to it instead of the arrow.
|
||||
const [askOffscreen, setAskOffscreen] = useState(false);
|
||||
const findAsk = useCallback(
|
||||
() =>
|
||||
answerableAskMessageId
|
||||
? (scrollRef.current?.querySelector<HTMLElement>(
|
||||
`[data-message-id="${CSS.escape(answerableAskMessageId)}"]`,
|
||||
) ?? null)
|
||||
: null,
|
||||
[answerableAskMessageId, scrollRef],
|
||||
);
|
||||
useEffect(() => {
|
||||
const root = scrollRef.current;
|
||||
const element = findAsk();
|
||||
if (!root || !element || typeof IntersectionObserver === "undefined") {
|
||||
setAskOffscreen(false);
|
||||
return;
|
||||
}
|
||||
const observer = new IntersectionObserver(
|
||||
(entries) => setAskOffscreen(!entries.some((entry) => entry.isIntersecting)),
|
||||
{ root, threshold: 0.2 },
|
||||
);
|
||||
observer.observe(element);
|
||||
return () => observer.disconnect();
|
||||
}, [findAsk, messages, scrollRef]);
|
||||
const jumpToAsk = useCallback(() => {
|
||||
const element = findAsk();
|
||||
if (!element) return;
|
||||
const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches;
|
||||
following.current = false;
|
||||
element.scrollIntoView({ behavior: reducedMotion ? "auto" : "smooth", block: "center" });
|
||||
element
|
||||
.querySelector<HTMLElement>("[data-ask-card] input, [data-ask-card] button")
|
||||
?.focus({ preventScroll: true });
|
||||
}, [findAsk]);
|
||||
const showJumpToLatest = !atEnd && !askOffscreen;
|
||||
|
||||
useLayoutEffect(() => {
|
||||
const button = jumpButtonRef.current;
|
||||
if (atEnd && button && document.activeElement === button) {
|
||||
@ -6209,6 +6243,7 @@ const Transcript = memo(function Transcript({
|
||||
)}
|
||||
<MessageView
|
||||
artifactTarget={artifactTarget}
|
||||
onOpenArtifact={onOpenArtifact}
|
||||
message={message}
|
||||
canAnswer={message.id === answerableAskMessageId}
|
||||
onOpenBot={onOpenBot}
|
||||
@ -6291,15 +6326,28 @@ const Transcript = memo(function Transcript({
|
||||
ref={jumpButtonRef}
|
||||
type="button"
|
||||
aria-label={t`Jump to latest`}
|
||||
aria-hidden={atEnd}
|
||||
tabIndex={atEnd ? -1 : 0}
|
||||
aria-hidden={!showJumpToLatest}
|
||||
tabIndex={showJumpToLatest ? 0 : -1}
|
||||
onClick={jumpToLatest}
|
||||
className={`absolute bottom-4 left-1/2 z-20 grid h-9 w-9 -translate-x-1/2 place-items-center rounded-full border border-border bg-muted/95 text-foreground/75 shadow-md backdrop-blur transition-[opacity,transform,background-color] duration-200 ease-[cubic-bezier(.22,1,.36,1)] hover:bg-border motion-reduce:transition-none ${
|
||||
atEnd ? "pointer-events-none translate-y-2 opacity-0" : "translate-y-0 opacity-100"
|
||||
showJumpToLatest
|
||||
? "translate-y-0 opacity-100"
|
||||
: "pointer-events-none translate-y-2 opacity-0"
|
||||
}`}
|
||||
>
|
||||
<ArrowDown size={17} strokeWidth={1.8} />
|
||||
</button>
|
||||
{askOffscreen ? (
|
||||
<button
|
||||
type="button"
|
||||
data-testid="needs-you-jump"
|
||||
onClick={jumpToAsk}
|
||||
className="absolute bottom-4 left-1/2 z-20 inline-flex h-9 -translate-x-1/2 items-center gap-1.5 rounded-full border border-border bg-muted/95 px-3.5 text-ui-body-sm font-medium text-foreground shadow-md backdrop-blur hover:bg-border"
|
||||
>
|
||||
<span aria-hidden="true" className="size-2 rounded-full bg-warning" />
|
||||
{t`Needs you`}
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
@ -7428,6 +7476,7 @@ function ComputerReleaseActions({
|
||||
|
||||
const MessageView = memo(function MessageView({
|
||||
artifactTarget,
|
||||
onOpenArtifact,
|
||||
canAnswer,
|
||||
message,
|
||||
onAnswer,
|
||||
@ -7449,6 +7498,7 @@ const MessageView = memo(function MessageView({
|
||||
encounterMode = false,
|
||||
}: {
|
||||
artifactTarget: ArtifactTarget;
|
||||
onOpenArtifact?: (artifact: WorkspaceArtifact) => void;
|
||||
canAnswer: boolean;
|
||||
message: ThreadMessage;
|
||||
onAnswer: (message: ThreadMessage, text: string) => Promise<void>;
|
||||
@ -7850,6 +7900,17 @@ const MessageView = memo(function MessageView({
|
||||
name={block.name}
|
||||
mimeType={block.mimeType}
|
||||
size={block.size}
|
||||
onOpenBeside={
|
||||
onOpenArtifact
|
||||
? () =>
|
||||
onOpenArtifact({
|
||||
messageId: message.id,
|
||||
artifactId: block.artifactId,
|
||||
name: block.name,
|
||||
mimeType: block.mimeType,
|
||||
})
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
|
||||
@ -38,10 +38,13 @@ export type NewSpaceConfirmInput = {
|
||||
|
||||
export function NewSpaceDialog({
|
||||
variant = "space",
|
||||
canLinkFolder,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: {
|
||||
variant?: "space" | "project";
|
||||
/** Folders live on the host, which only the deployment owner may link. */
|
||||
canLinkFolder: boolean;
|
||||
onCancel: () => void;
|
||||
onConfirm: (input: NewSpaceConfirmInput) => Promise<void>;
|
||||
}) {
|
||||
@ -53,6 +56,7 @@ export function NewSpaceDialog({
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const isProject = variant === "project";
|
||||
const showFolder = isProject && canLinkFolder;
|
||||
const desktop = desktopBridge();
|
||||
const trimmedWorkspacePath = workspacePath.trim();
|
||||
|
||||
@ -63,7 +67,7 @@ export function NewSpaceDialog({
|
||||
setError(null);
|
||||
void onConfirm({
|
||||
name: trimmed,
|
||||
...(isProject && trimmedWorkspacePath ? { workspacePath: trimmedWorkspacePath } : {}),
|
||||
...(showFolder && trimmedWorkspacePath ? { workspacePath: trimmedWorkspacePath } : {}),
|
||||
}).catch((reason: unknown) => {
|
||||
setError(
|
||||
reason instanceof Error
|
||||
@ -104,7 +108,7 @@ export function NewSpaceDialog({
|
||||
className="mt-2"
|
||||
/>
|
||||
</label>
|
||||
{isProject ? (
|
||||
{showFolder ? (
|
||||
<div className="space-y-2">
|
||||
<label htmlFor={folderId} className="block text-ui-body text-foreground/75">
|
||||
<Trans>Project folder</Trans>
|
||||
|
||||
@ -240,6 +240,26 @@ body,
|
||||
-webkit-app-region: no-drag;
|
||||
}
|
||||
|
||||
/* In-app browser load indicator: a short bar sweeping under the address bar. */
|
||||
.rk-browser-progress {
|
||||
animation: rkBrowserProgress 1.1s cubic-bezier(0.65, 0, 0.35, 1) infinite;
|
||||
}
|
||||
@keyframes rkBrowserProgress {
|
||||
from {
|
||||
transform: translateX(-100%);
|
||||
}
|
||||
to {
|
||||
transform: translateX(300%);
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.rk-browser-progress {
|
||||
animation: none;
|
||||
width: 100%;
|
||||
opacity: 0.6;
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes rkPulse {
|
||||
0%,
|
||||
100% {
|
||||
|
||||
32
docs/ADAPTIVE-WORKSPACE.md
Normal file
32
docs/ADAPTIVE-WORKSPACE.md
Normal file
@ -0,0 +1,32 @@
|
||||
# Adaptive workspace
|
||||
|
||||
The pane beside a conversation follows what the bot is actually doing, not a guess about the task.
|
||||
No classifier or model decides the layout: `deriveWorkspaceFocus` in `@rakazo/core` reads the thread
|
||||
state every client already has, so web and mobile agree and the rules stay testable offline.
|
||||
|
||||
## Signals
|
||||
|
||||
| Focus | Signal | Web | Mobile |
|
||||
|-------|--------|-----|--------|
|
||||
| Screen | A fresh `thread.computer` pointer event from a working run, or a takeover request | Live screen opens beside the chat (wide pane), steps back a few seconds after the work ends | Pill above the composer opens the computer |
|
||||
| Artifact | The latest run attached a file with `present: true` (`attach_file`) | File opens beside the chat until the user closes it or the bot presents another | Unchanged: file cards open the native preview |
|
||||
| Needs you | An unanswered ask on a waiting run | "Needs you" pill replaces the jump arrow when the question scrolled out of view | Existing push and attention notifications |
|
||||
|
||||
Everything else stays a plain conversation.
|
||||
|
||||
## Rules
|
||||
|
||||
- The pane moves only on focus edges (`applyWorkspaceFocus`). The first focus of a conversation is a
|
||||
baseline, so opening a thread never jumps.
|
||||
- A pane the user opened by hand is never replaced or closed automatically. Auto panes close only
|
||||
when their own focus ends; a presented file stays.
|
||||
- Windows narrower than 1024px keep the conversation full width; file cards fall back to the dialog.
|
||||
- Per bot, the client learns from the user: opening the screen by hand while the bot works three
|
||||
times makes it open on every run; closing an automatically opened screen twice stops it opening
|
||||
on its own. A takeover request always shows the screen.
|
||||
- Pane widths are remembered per device.
|
||||
|
||||
## Not in scope
|
||||
|
||||
Generated layouts, per-domain dashboards and prompt classification stay out: they guess, move the UI
|
||||
under the user, and cannot be verified deterministically.
|
||||
@ -412,10 +412,39 @@ EVAL_HELPERS = r"""
|
||||
})
|
||||
.join("\n");
|
||||
}
|
||||
// Screen pixels of the element center's box, only while that center is in the viewport.
|
||||
// Drives the visible bot cursor; the model never sees it.
|
||||
function screenBox(el, viewport) {
|
||||
const rect = el.getBoundingClientRect();
|
||||
const cx = rect.left + rect.width / 2;
|
||||
const cy = rect.top + rect.height / 2;
|
||||
if (cx < 0 || cy < 0 || cx > window.innerWidth || cy > window.innerHeight) return undefined;
|
||||
const box = [
|
||||
viewport.x + rect.left * viewport.scale,
|
||||
viewport.y + rect.top * viewport.scale,
|
||||
rect.width * viewport.scale,
|
||||
rect.height * viewport.scale,
|
||||
].map(Math.round);
|
||||
return box[0] >= 0 && box[1] >= 0 && box[2] > 0 && box[3] > 0 ? box : undefined;
|
||||
}
|
||||
function viewportOnScreen() {
|
||||
const scale = window.devicePixelRatio || 1;
|
||||
// Browser chrome sits above the page; side borders are split evenly.
|
||||
return {
|
||||
scale,
|
||||
x: (window.screenX + Math.max(0, (window.outerWidth - window.innerWidth) / 2)) * scale,
|
||||
y: (window.screenY + Math.max(0, window.outerHeight - window.innerHeight)) * scale,
|
||||
screen: {
|
||||
width: Math.round(window.screen.width * scale),
|
||||
height: Math.round(window.screen.height * scale),
|
||||
},
|
||||
};
|
||||
}
|
||||
window.__rakazoPageBrowser = {
|
||||
snapshot() {
|
||||
refMap.clear();
|
||||
const elements = [];
|
||||
const viewport = viewportOnScreen();
|
||||
for (const el of document.querySelectorAll("body *")) {
|
||||
if (!isInteractive(el) || !visible(el)) continue;
|
||||
const ref = newRef();
|
||||
@ -428,12 +457,14 @@ EVAL_HELPERS = r"""
|
||||
: ["checkbox", "radio"].includes(el.type) ? (el.checked ? "checked" : "unchecked")
|
||||
: "value" in el ? String(el.value ?? "").slice(0, 2000) : undefined,
|
||||
tag: el.tagName.toLowerCase(),
|
||||
box: screenBox(el, viewport),
|
||||
});
|
||||
if (elements.length >= 80) break;
|
||||
}
|
||||
return {
|
||||
url: location.href,
|
||||
title: (document.title || "").slice(0, 200),
|
||||
screen: viewport.screen,
|
||||
elements,
|
||||
tree: (document.body?.innerText || "").slice(0, 12000) + "\n" + formatTree(elements)
|
||||
+ (document.querySelector("iframe,frame") ? "\nEmbedded frames require desktop tools." : ""),
|
||||
|
||||
47
infra/updater/src/ci-workflow.test.ts
Normal file
47
infra/updater/src/ci-workflow.test.ts
Normal file
@ -0,0 +1,47 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
|
||||
const workflowText = readFileSync(
|
||||
path.resolve(import.meta.dirname, "../../../.github/workflows/ci.yml"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
interface WorkflowJob {
|
||||
if?: string;
|
||||
needs?: string[];
|
||||
"continue-on-error"?: boolean;
|
||||
}
|
||||
|
||||
const workflow = parse(workflowText) as {
|
||||
jobs: Record<string, WorkflowJob>;
|
||||
};
|
||||
|
||||
describe("ci workflow merge gate", () => {
|
||||
it("aggregates only product jobs in Product CI", () => {
|
||||
const productGate = workflow.jobs["product-gate"];
|
||||
expect(productGate?.needs).toEqual([
|
||||
"lint",
|
||||
"check",
|
||||
"build",
|
||||
"test",
|
||||
"test-integration",
|
||||
"test-e2e",
|
||||
]);
|
||||
expect(productGate?.if).toBe("always()");
|
||||
});
|
||||
|
||||
it("soft-fails mobile OTA so missing Expo credentials do not fail the workflow", () => {
|
||||
const mobile = workflow.jobs["publish-mobile-update"];
|
||||
expect(mobile?.["continue-on-error"]).toBe(true);
|
||||
expect(workflowText).toContain('echo "ready=false" >> "$GITHUB_OUTPUT"');
|
||||
expect(workflowText).not.toMatch(
|
||||
/Check Expo automation token[\s\S]*?exit 1[\s\S]*?Require Expo/,
|
||||
);
|
||||
expect(workflowText).not.toContain(
|
||||
"Add an EXPO_TOKEN repository secret with access to the inbox-zero/rakazo Expo project.",
|
||||
);
|
||||
expect(workflowText).toContain("steps.expo_token.outputs.ready == 'true'");
|
||||
});
|
||||
});
|
||||
@ -43,6 +43,8 @@ describe("server image publish workflow", () => {
|
||||
expect(workflowText).toContain("actions/attest-build-provenance@");
|
||||
expect(workflowText).not.toContain("github.repository_owner_type");
|
||||
expect(workflowText).toContain(`gh api "repos/\${GITHUB_REPOSITORY}"`);
|
||||
// `gh api --jq` already prints raw strings; `--jq -r` takes `-r` as the filter and fails.
|
||||
expect(workflowText).not.toContain("--jq -r");
|
||||
expect(workflowText).toContain(".owner.type");
|
||||
expect(workflowText).toContain("if: steps.attestation.outputs.enabled == 'true'");
|
||||
expect(workflowText).toContain('[[ "$owner_type" == "User" && "$private" == "true" ]]');
|
||||
|
||||
@ -746,14 +746,21 @@ export interface BrowserSnapshotNode {
|
||||
name: string;
|
||||
value?: string;
|
||||
tag?: string;
|
||||
/** Screen pixels when the element's center is in view; drives the visible bot cursor only. */
|
||||
box?: BrowserScreenBox;
|
||||
}
|
||||
|
||||
/** `[x, y, width, height]` in screen pixels of the computer display. */
|
||||
export type BrowserScreenBox = [number, number, number, number];
|
||||
|
||||
export interface BrowserSnapshotResult {
|
||||
url: string;
|
||||
title: string;
|
||||
/** Compact accessibility-style tree for the model. */
|
||||
tree: string;
|
||||
elements: BrowserSnapshotNode[];
|
||||
/** Display size the element boxes are relative to. */
|
||||
screen?: { width: number; height: number };
|
||||
fallback?: "computer_act";
|
||||
error?: string;
|
||||
}
|
||||
@ -779,6 +786,7 @@ export interface BrowserActResult {
|
||||
/** Snapshot after the actions when available. */
|
||||
tree?: string;
|
||||
elements?: BrowserSnapshotNode[];
|
||||
screen?: { width: number; height: number };
|
||||
fallback?: "computer_act";
|
||||
error?: string;
|
||||
}
|
||||
|
||||
@ -5,6 +5,14 @@ import type {
|
||||
BrowserProvider,
|
||||
ComputerRef,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { ComputerPointerPayload } from "@rakazo/contracts";
|
||||
import { type BrowserPointerTargets, withoutBrowserGeometry } from "./computer-pointer.js";
|
||||
|
||||
/** Optional hooks that let the visible bot cursor follow page-browser actions. */
|
||||
export interface BrowserPointerHooks {
|
||||
targets: BrowserPointerTargets;
|
||||
announce: (payload: ComputerPointerPayload) => Promise<void>;
|
||||
}
|
||||
|
||||
const MAX_BROWSER_ACTIONS = 24;
|
||||
|
||||
@ -13,6 +21,7 @@ export async function browserNavigateFromTool(
|
||||
computer: ComputerRef,
|
||||
context: AdapterContext,
|
||||
args: Record<string, unknown>,
|
||||
pointer?: BrowserPointerHooks,
|
||||
) {
|
||||
const url = String(args.url ?? "").trim();
|
||||
try {
|
||||
@ -23,6 +32,7 @@ export async function browserNavigateFromTool(
|
||||
fallback: "computer_act" as const,
|
||||
};
|
||||
}
|
||||
pointer?.targets.forget();
|
||||
const result = await browser.navigate(computer, { url, signal: context.signal }, context);
|
||||
return formatBrowserResult(result);
|
||||
} catch (error) {
|
||||
@ -39,11 +49,13 @@ export async function browserSnapshotFromTool(
|
||||
computer: ComputerRef,
|
||||
context: AdapterContext,
|
||||
args: Record<string, unknown>,
|
||||
pointer?: BrowserPointerHooks,
|
||||
) {
|
||||
void args;
|
||||
try {
|
||||
const result = await browser.snapshot(computer, { signal: context.signal }, context);
|
||||
return formatBrowserResult(result);
|
||||
pointer?.targets.remember(result);
|
||||
return formatBrowserResult(withoutBrowserGeometry(result));
|
||||
} catch (error) {
|
||||
context.signal.throwIfAborted();
|
||||
return {
|
||||
@ -58,11 +70,16 @@ export async function browserActFromTool(
|
||||
computer: ComputerRef,
|
||||
context: AdapterContext,
|
||||
args: Record<string, unknown>,
|
||||
pointer?: BrowserPointerHooks,
|
||||
) {
|
||||
try {
|
||||
const actions = parseBrowserActions(args.actions);
|
||||
const announcement = pointer?.targets.payloadFor(actions);
|
||||
if (announcement) await pointer?.announce(announcement);
|
||||
const result = await browser.act(computer, { actions, signal: context.signal }, context);
|
||||
return formatBrowserResult(result);
|
||||
if (result.elements) pointer?.targets.remember(result);
|
||||
else pointer?.targets.forget();
|
||||
return formatBrowserResult(withoutBrowserGeometry(result));
|
||||
} catch (error) {
|
||||
context.signal.throwIfAborted();
|
||||
return {
|
||||
|
||||
@ -144,10 +144,17 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
{
|
||||
name: "attach_file",
|
||||
description:
|
||||
"Attach a workspace file from this bot's home to the chat thread as an image or common file. The file stays in place; users can open it from the message.",
|
||||
"Attach a workspace file from this bot's home to the chat thread as an image or common file. The file stays in place; users can open it from the message. Set present when the file is the result the user asked for, so it opens beside the conversation.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { path: { type: "string" } },
|
||||
properties: {
|
||||
path: { type: "string" },
|
||||
present: {
|
||||
type: "boolean",
|
||||
description:
|
||||
"True only for the finished result of the request, not for supporting files.",
|
||||
},
|
||||
},
|
||||
required: ["path"],
|
||||
},
|
||||
},
|
||||
|
||||
@ -115,6 +115,7 @@ export class ComputerBrowserProvider implements BrowserProvider {
|
||||
title: typeof live.title === "string" ? live.title : "",
|
||||
tree: typeof live.tree === "string" ? live.tree : formatTree(elements),
|
||||
elements,
|
||||
...(live.screen ? { screen: live.screen } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@ -150,6 +151,7 @@ export class ComputerBrowserProvider implements BrowserProvider {
|
||||
title: typeof live.title === "string" ? live.title : "",
|
||||
tree: typeof live.tree === "string" ? live.tree : elements ? formatTree(elements) : undefined,
|
||||
elements,
|
||||
...(live.screen ? { screen: live.screen } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
164
packages/adapters/src/computer-pointer.test.ts
Normal file
164
packages/adapters/src/computer-pointer.test.ts
Normal file
@ -0,0 +1,164 @@
|
||||
import type {
|
||||
AdapterContext,
|
||||
BrowserActRequest,
|
||||
BrowserActResult,
|
||||
BrowserProvider,
|
||||
BrowserSnapshotResult,
|
||||
ComputerRef,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import { ComputerPointerPayload } from "@rakazo/contracts";
|
||||
import { DEFAULT_COMPUTER_SCREEN } from "@rakazo/core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { browserActFromTool, browserSnapshotFromTool } from "./browser-tools.js";
|
||||
import {
|
||||
BrowserPointerTargets,
|
||||
computerPointerPayload,
|
||||
computerPointerSteps,
|
||||
} from "./computer-pointer.js";
|
||||
import { parseComputerActions } from "./computer-tools.js";
|
||||
|
||||
const context: AdapterContext = {
|
||||
operationId: "1",
|
||||
traceId: "1",
|
||||
spaceId: "w",
|
||||
userId: "u",
|
||||
signal: new AbortController().signal,
|
||||
};
|
||||
const computer: ComputerRef = { id: "c1", botId: "b1", kind: "docker", providerRef: "c1" };
|
||||
|
||||
describe("computer pointer steps", () => {
|
||||
it("maps a computer_act batch to cursor steps in order", () => {
|
||||
const steps = computerPointerSteps(
|
||||
parseComputerActions([
|
||||
{ kind: "move", x: 10, y: 20 },
|
||||
{ kind: "click", x: 30, y: 40, double: true },
|
||||
{ kind: "click", x: 50, y: 60, button: "right" },
|
||||
{ kind: "type", text: "hello" },
|
||||
{ kind: "key", key: "Enter" },
|
||||
{ kind: "wait", ms: 100 },
|
||||
{ kind: "scroll", direction: "down" },
|
||||
{ kind: "down", x: 1, y: 2 },
|
||||
{ kind: "up", x: 3, y: 4 },
|
||||
]),
|
||||
);
|
||||
expect(steps).toEqual([
|
||||
{ action: "move", x: 10, y: 20 },
|
||||
{ action: "double_click", x: 30, y: 40 },
|
||||
{ action: "right_click", x: 50, y: 60 },
|
||||
{ action: "type" },
|
||||
{ action: "key" },
|
||||
{ action: "scroll" },
|
||||
{ action: "down", x: 1, y: 2 },
|
||||
{ action: "up", x: 3, y: 4 },
|
||||
]);
|
||||
const payload = computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps);
|
||||
expect(ComputerPointerPayload.parse(payload)).toEqual(payload);
|
||||
});
|
||||
|
||||
it("keeps two separate clicks at different points apart", () => {
|
||||
expect(
|
||||
computerPointerSteps(
|
||||
parseComputerActions([
|
||||
{ kind: "click", x: 1, y: 1 },
|
||||
{ kind: "click", x: 2, y: 2 },
|
||||
]),
|
||||
),
|
||||
).toEqual([
|
||||
{ action: "click", x: 1, y: 1 },
|
||||
{ action: "click", x: 2, y: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("announces nothing for a batch without visible steps", () => {
|
||||
const steps = computerPointerSteps(parseComputerActions([{ kind: "wait", ms: 10 }]));
|
||||
expect(computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser pointer targets", () => {
|
||||
it("points at element centers and stops at the first unknown ref", () => {
|
||||
const targets = new BrowserPointerTargets();
|
||||
targets.remember({
|
||||
screen: { width: 1280, height: 800 },
|
||||
elements: [
|
||||
{ ref: "e1", role: "textbox", name: "Name", box: [100, 200, 40, 20] },
|
||||
{ ref: "e2", role: "button", name: "Save", box: [300, 400, 11, 11] },
|
||||
{ ref: "e3", role: "button", name: "Hidden" },
|
||||
// Untrusted helper output never becomes a cursor target.
|
||||
{ ref: "e4", role: "button", name: "Bad", box: [-5, 0, 10, 10] as never },
|
||||
],
|
||||
});
|
||||
expect(
|
||||
targets.payloadFor([
|
||||
{ kind: "fill", ref: "e1", text: "x" },
|
||||
{ kind: "click", ref: "e2" },
|
||||
{ kind: "click", ref: "e3" },
|
||||
{ kind: "click", ref: "e1" },
|
||||
]),
|
||||
).toEqual({
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [
|
||||
{ action: "type", x: 120, y: 210 },
|
||||
{ action: "click", x: 306, y: 406 },
|
||||
],
|
||||
});
|
||||
expect(targets.payloadFor([{ kind: "click", ref: "e4" }])).toBeUndefined();
|
||||
targets.forget();
|
||||
expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
|
||||
});
|
||||
|
||||
it("needs a reported screen size", () => {
|
||||
const targets = new BrowserPointerTargets();
|
||||
targets.remember({ elements: [{ ref: "e1", role: "button", name: "A", box: [1, 1, 2, 2] }] });
|
||||
expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("page browser tools with a visible cursor", () => {
|
||||
it("announces before acting and keeps geometry away from the model", async () => {
|
||||
const order: string[] = [];
|
||||
const snapshot: BrowserSnapshotResult = {
|
||||
url: "https://example.test/",
|
||||
title: "Example",
|
||||
tree: '- button "Save" [e1]',
|
||||
screen: { width: 1280, height: 800 },
|
||||
elements: [{ ref: "e1", role: "button", name: "Save", box: [10, 20, 30, 40] }],
|
||||
};
|
||||
const browser = {
|
||||
snapshot: async () => snapshot,
|
||||
act: async (_computer: ComputerRef, request: BrowserActRequest) => {
|
||||
order.push(`act:${request.actions.length}`);
|
||||
return { ok: true, completed: 1, ...snapshot } satisfies BrowserActResult;
|
||||
},
|
||||
} as unknown as BrowserProvider;
|
||||
const announced: unknown[] = [];
|
||||
const pointer = {
|
||||
targets: new BrowserPointerTargets(),
|
||||
announce: async (payload: ComputerPointerPayload) => {
|
||||
order.push("announce");
|
||||
announced.push(payload);
|
||||
},
|
||||
};
|
||||
|
||||
const seen = await browserSnapshotFromTool(browser, computer, context, {}, pointer);
|
||||
expect(JSON.stringify(seen)).not.toMatch(/box|screen/);
|
||||
|
||||
const acted = await browserActFromTool(
|
||||
browser,
|
||||
computer,
|
||||
context,
|
||||
{ actions: [{ kind: "click", ref: "e1" }] },
|
||||
pointer,
|
||||
);
|
||||
expect(order).toEqual(["announce", "act:1"]);
|
||||
expect(announced).toEqual([
|
||||
{
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [{ action: "click", x: 25, y: 40 }],
|
||||
},
|
||||
]);
|
||||
expect(JSON.stringify(acted)).not.toMatch(/box|screen/);
|
||||
});
|
||||
});
|
||||
130
packages/adapters/src/computer-pointer.ts
Normal file
130
packages/adapters/src/computer-pointer.ts
Normal file
@ -0,0 +1,130 @@
|
||||
import type {
|
||||
BrowserActStep,
|
||||
BrowserScreenBox,
|
||||
BrowserSnapshotNode,
|
||||
ComputerAction,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import {
|
||||
type ComputerPointerPayload,
|
||||
type ComputerPointerStep,
|
||||
MAX_COMPUTER_POINTER_STEPS,
|
||||
} from "@rakazo/contracts";
|
||||
|
||||
export type ComputerScreenSize = ComputerPointerPayload["screen"];
|
||||
|
||||
/** Pointer steps for a validated computer_act batch, in execution order. */
|
||||
export function computerPointerSteps(actions: ComputerAction[]): ComputerPointerStep[] {
|
||||
const steps: ComputerPointerStep[] = [];
|
||||
for (let index = 0; index < actions.length; index += 1) {
|
||||
const action = actions[index]!;
|
||||
if (action.kind === "pointer") {
|
||||
const next = actions[index + 1];
|
||||
// parseComputerActions expands a double click into two identical clicks.
|
||||
if (
|
||||
action.type === "click" &&
|
||||
next?.kind === "pointer" &&
|
||||
next.type === "click" &&
|
||||
next.x === action.x &&
|
||||
next.y === action.y &&
|
||||
next.button === action.button
|
||||
) {
|
||||
steps.push({ action: "double_click", x: action.x, y: action.y });
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
const kind =
|
||||
action.type === "click"
|
||||
? action.button === "right"
|
||||
? "right_click"
|
||||
: "click"
|
||||
: action.type;
|
||||
steps.push({ action: kind, x: action.x, y: action.y });
|
||||
} else if (action.kind === "clipboard") {
|
||||
steps.push({ action: "type" });
|
||||
} else if (action.kind === "key" || action.kind === "scroll") {
|
||||
steps.push({ action: action.kind });
|
||||
}
|
||||
}
|
||||
return steps.slice(0, MAX_COMPUTER_POINTER_STEPS);
|
||||
}
|
||||
|
||||
export function computerPointerPayload(
|
||||
screen: ComputerScreenSize,
|
||||
steps: ComputerPointerStep[],
|
||||
): ComputerPointerPayload | undefined {
|
||||
if (steps.length === 0) return undefined;
|
||||
return { kind: "pointer", screen, steps: steps.slice(0, MAX_COMPUTER_POINTER_STEPS) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Remembers where the page browser last saw each element on screen, so a browser_act batch
|
||||
* can be announced as pointer steps before it runs. Geometry never reaches the model.
|
||||
*/
|
||||
export class BrowserPointerTargets {
|
||||
private boxes = new Map<string, BrowserScreenBox>();
|
||||
private screen: ComputerScreenSize | undefined;
|
||||
|
||||
remember(result: { elements?: BrowserSnapshotNode[]; screen?: ComputerScreenSize }) {
|
||||
if (!Array.isArray(result.elements)) return;
|
||||
this.boxes = new Map();
|
||||
this.screen = validScreen(result.screen);
|
||||
for (const element of result.elements) {
|
||||
if (validBox(element.box)) this.boxes.set(element.ref, element.box);
|
||||
}
|
||||
}
|
||||
|
||||
forget() {
|
||||
this.boxes = new Map();
|
||||
this.screen = undefined;
|
||||
}
|
||||
|
||||
payloadFor(actions: BrowserActStep[]): ComputerPointerPayload | undefined {
|
||||
if (!this.screen) return undefined;
|
||||
const steps: ComputerPointerStep[] = [];
|
||||
for (const action of actions) {
|
||||
const box = this.boxes.get(action.ref);
|
||||
// Later steps may target elements an earlier step reveals; stop at the first unknown one.
|
||||
if (!box) break;
|
||||
const [x, y, width, height] = box;
|
||||
steps.push({
|
||||
action: action.kind === "click" ? "click" : "type",
|
||||
x: Math.round(x + width / 2),
|
||||
y: Math.round(y + height / 2),
|
||||
});
|
||||
}
|
||||
return computerPointerPayload(this.screen, steps);
|
||||
}
|
||||
}
|
||||
|
||||
/** Strips screen geometry from page-browser results before they reach the model. */
|
||||
export function withoutBrowserGeometry<T extends { elements?: BrowserSnapshotNode[] }>(
|
||||
result: T,
|
||||
): Omit<T, "screen"> {
|
||||
const { screen: _screen, ...rest } = result as T & { screen?: unknown };
|
||||
if (!Array.isArray(rest.elements)) return rest;
|
||||
return {
|
||||
...rest,
|
||||
elements: rest.elements.map(({ box: _box, ...element }) => element),
|
||||
};
|
||||
}
|
||||
|
||||
function validScreen(value: unknown): ComputerScreenSize | undefined {
|
||||
if (!value || typeof value !== "object") return undefined;
|
||||
const { width, height } = value as Record<string, unknown>;
|
||||
if (!isPixel(width) || !isPixel(height) || width === 0 || height === 0) return undefined;
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function validBox(value: unknown): value is BrowserScreenBox {
|
||||
return (
|
||||
Array.isArray(value) &&
|
||||
value.length === 4 &&
|
||||
value.every(isPixel) &&
|
||||
(value[2] as number) > 0 &&
|
||||
(value[3] as number) > 0
|
||||
);
|
||||
}
|
||||
|
||||
function isPixel(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100_000;
|
||||
}
|
||||
@ -25,7 +25,7 @@ import {
|
||||
routineWakeupJob,
|
||||
runContinueJob,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { ModelTier, RunStatus } from "@rakazo/contracts";
|
||||
import type { ComputerPointerPayload, ModelTier, RunStatus } from "@rakazo/contracts";
|
||||
import {
|
||||
ATTACHMENT_MAX_BYTES,
|
||||
BOT_DESCRIPTION_MAX_LENGTH,
|
||||
@ -45,6 +45,7 @@ import {
|
||||
applyJudgeDecision,
|
||||
assertTransition,
|
||||
botMessageAllowsSilence,
|
||||
computerScreenSize,
|
||||
connectorKindFromToolName,
|
||||
containsSecret,
|
||||
createStreamingRedactor,
|
||||
@ -165,6 +166,7 @@ import {
|
||||
} from "./bot-secrets.js";
|
||||
import { createBrowserProvider } from "./browser-provider-factory.js";
|
||||
import {
|
||||
type BrowserPointerHooks,
|
||||
browserActFromTool,
|
||||
browserNavigateFromTool,
|
||||
browserSnapshotFromTool,
|
||||
@ -193,6 +195,11 @@ import {
|
||||
renewComputerExecutionLease,
|
||||
screenLeaseIdForRun,
|
||||
} from "./computer-lifecycle.js";
|
||||
import {
|
||||
BrowserPointerTargets,
|
||||
computerPointerPayload,
|
||||
computerPointerSteps,
|
||||
} from "./computer-pointer.js";
|
||||
import { withComputerScreenAvailability } from "./computer-screens.js";
|
||||
import {
|
||||
assertTeamShellCommandAllowed,
|
||||
@ -1653,6 +1660,27 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
lastComputerFrameId = observation.frameId;
|
||||
return result;
|
||||
};
|
||||
// Best-effort: the visible cursor must never change a tool result or strand the run.
|
||||
const announcePointer = async (payload: ComputerPointerPayload) => {
|
||||
try {
|
||||
await deps.events.append({
|
||||
spaceId: run.spaceId,
|
||||
threadId: thread.id,
|
||||
botId: bot.id,
|
||||
type: "thread.computer",
|
||||
runId,
|
||||
payload,
|
||||
});
|
||||
} catch (error) {
|
||||
getLogger().warn("computer pointer append failed", {
|
||||
error: sanitizeConnectorError(error, runSecrets),
|
||||
});
|
||||
}
|
||||
};
|
||||
const browserPointer: BrowserPointerHooks = {
|
||||
targets: new BrowserPointerTargets(),
|
||||
announce: announcePointer,
|
||||
};
|
||||
|
||||
const pauseForApproval = () => {
|
||||
approvalPausePending = true;
|
||||
@ -2208,10 +2236,16 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
}
|
||||
workspaceCheckpoint.markDirty();
|
||||
return computerScreenToolResult(async () => {
|
||||
const actions = parseComputerActions(args.actions);
|
||||
const pointer = computerPointerPayload(
|
||||
computerScreenSize(computer.kind),
|
||||
computerPointerSteps(actions),
|
||||
);
|
||||
if (pointer) await announcePointer(pointer);
|
||||
const result = await deps.sandbox.act(
|
||||
computer,
|
||||
{
|
||||
actions: parseComputerActions(args.actions),
|
||||
actions,
|
||||
observe: args.observe !== false,
|
||||
settleMs: Number(args.settle_ms ?? 350),
|
||||
},
|
||||
@ -2415,6 +2449,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
filePath,
|
||||
bytes,
|
||||
operationId: executionId,
|
||||
present: args.present === true,
|
||||
},
|
||||
);
|
||||
await publishMessage(deps, run, "bot", [attached.block]);
|
||||
@ -2595,7 +2630,10 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
: name === "browser_snapshot"
|
||||
? browserSnapshotFromTool
|
||||
: browserActFromTool;
|
||||
return computerScreenToolResult(() => tool(browser, computer, context, args), finish);
|
||||
return computerScreenToolResult(
|
||||
() => tool(browser, computer, context, args, browserPointer),
|
||||
finish,
|
||||
);
|
||||
}
|
||||
|
||||
if (name.startsWith("cloud_agent_")) {
|
||||
|
||||
60
packages/adapters/src/project-workspace-files.test.ts
Normal file
60
packages/adapters/src/project-workspace-files.test.ts
Normal file
@ -0,0 +1,60 @@
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { listProjectWorkspaceFiles, readProjectWorkspaceFile } from "./project-workspace-files.js";
|
||||
|
||||
describe("project workspace files", () => {
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function linkedFolderBesideSecret() {
|
||||
const base = mkdtempSync(path.join(tmpdir(), "project-workspace-files-"));
|
||||
roots.push(base);
|
||||
const project = path.join(base, "project");
|
||||
const outside = path.join(base, "outside");
|
||||
mkdirSync(path.join(project, "docs"), { recursive: true });
|
||||
mkdirSync(outside);
|
||||
writeFileSync(path.join(project, "docs", "readme.md"), "inside");
|
||||
writeFileSync(path.join(outside, "secret.txt"), "outside");
|
||||
return { project, outside };
|
||||
}
|
||||
|
||||
it("lists and reads files inside the linked folder, including symlinks that stay inside", async () => {
|
||||
const { project } = linkedFolderBesideSecret();
|
||||
symlinkSync(path.join(project, "docs", "readme.md"), path.join(project, "readme-link.md"));
|
||||
|
||||
const listed = await listProjectWorkspaceFiles(project, "project");
|
||||
expect(listed.map((entry) => entry.path).sort()).toEqual([
|
||||
"project/docs",
|
||||
"project/readme-link.md",
|
||||
]);
|
||||
expect(await listProjectWorkspaceFiles(project, "project/docs")).toEqual([
|
||||
{ path: "project/docs/readme.md", kind: "file", size: 6 },
|
||||
]);
|
||||
const read = await readProjectWorkspaceFile(project, "project/readme-link.md");
|
||||
expect(new TextDecoder().decode(read)).toBe("inside");
|
||||
});
|
||||
|
||||
it("keeps symlinks from leaving the linked folder", async () => {
|
||||
const { project, outside } = linkedFolderBesideSecret();
|
||||
symlinkSync(outside, path.join(project, "escape"));
|
||||
symlinkSync(path.join(outside, "secret.txt"), path.join(project, "secret-link.txt"));
|
||||
symlinkSync(path.join(project, "missing"), path.join(project, "dangling"));
|
||||
|
||||
const listed = await listProjectWorkspaceFiles(project, "project");
|
||||
expect(listed.map((entry) => entry.path)).toEqual(["project/docs"]);
|
||||
await expect(listProjectWorkspaceFiles(project, "project/escape")).rejects.toThrow(/escapes/i);
|
||||
await expect(readProjectWorkspaceFile(project, "project/escape/secret.txt")).rejects.toThrow(
|
||||
/escapes/i,
|
||||
);
|
||||
await expect(readProjectWorkspaceFile(project, "project/secret-link.txt")).rejects.toThrow(
|
||||
/escapes/i,
|
||||
);
|
||||
});
|
||||
});
|
||||
@ -1,6 +1,8 @@
|
||||
import { readdir, readFile, stat } from "node:fs/promises";
|
||||
import { readdir, readFile, realpath, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { ComputerFileEntry } from "@rakazo/adapter-kit";
|
||||
import { normalizeWorkspacePath } from "./computer-support.js";
|
||||
import { isAllowedDesktopPath } from "./desktop-sandbox-paths.js";
|
||||
import {
|
||||
isProjectWorkspacePath,
|
||||
projectWorkspaceRelativePath,
|
||||
@ -14,28 +16,34 @@ export async function listProjectWorkspaceFiles(
|
||||
if (!isProjectWorkspacePath(storedPath)) {
|
||||
throw new Error("Path is not under the project workspace");
|
||||
}
|
||||
const hostDirectory = resolveProjectHostPath(workspaceRoot, storedPath);
|
||||
const root = await realpath(workspaceRoot);
|
||||
const hostDirectory = await containedHostPath(
|
||||
root,
|
||||
resolveProjectHostPath(workspaceRoot, storedPath),
|
||||
);
|
||||
const relativeDirectory = projectWorkspaceRelativePath(storedPath);
|
||||
const entries = await readdir(hostDirectory, { withFileTypes: true });
|
||||
const listed = await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
const childHost = resolveProjectHostPath(
|
||||
workspaceRoot,
|
||||
relativeDirectory ? `project/${relativeDirectory}/${entry.name}` : `project/${entry.name}`,
|
||||
);
|
||||
const info = await stat(childHost);
|
||||
// A symlink that leaves the linked folder, or leads nowhere, is not part of the project.
|
||||
const info = await containedHostPath(root, path.join(hostDirectory, entry.name))
|
||||
.then((target) => stat(target))
|
||||
.catch(() => null);
|
||||
if (!info) return [];
|
||||
const childStored = normalizeWorkspacePath(
|
||||
relativeDirectory ? `project/${relativeDirectory}/${entry.name}` : `project/${entry.name}`,
|
||||
);
|
||||
return {
|
||||
path: childStored,
|
||||
kind: info.isDirectory() ? ("dir" as const) : ("file" as const),
|
||||
size: info.size,
|
||||
...(info.isFile() && info.mode & 0o100 ? { executable: true } : {}),
|
||||
};
|
||||
return [
|
||||
{
|
||||
path: childStored,
|
||||
kind: info.isDirectory() ? ("dir" as const) : ("file" as const),
|
||||
size: info.size,
|
||||
...(info.isFile() && info.mode & 0o100 ? { executable: true } : {}),
|
||||
},
|
||||
];
|
||||
}),
|
||||
);
|
||||
return listed;
|
||||
return listed.flat();
|
||||
}
|
||||
|
||||
export async function readProjectWorkspaceFile(
|
||||
@ -43,10 +51,22 @@ export async function readProjectWorkspaceFile(
|
||||
storedPath: string,
|
||||
options?: { maxBytes?: number },
|
||||
): Promise<Uint8Array> {
|
||||
const hostPath = resolveProjectHostPath(workspaceRoot, storedPath);
|
||||
const hostPath = await containedHostPath(
|
||||
await realpath(workspaceRoot),
|
||||
resolveProjectHostPath(workspaceRoot, storedPath),
|
||||
);
|
||||
const info = await stat(hostPath);
|
||||
if (options?.maxBytes !== undefined && info.size > options.maxBytes) {
|
||||
throw new Error(`computer file exceeds ${options.maxBytes} bytes`);
|
||||
}
|
||||
return new Uint8Array(await readFile(hostPath));
|
||||
}
|
||||
|
||||
/** Follows symlinks, but only to places inside the linked folder. */
|
||||
async function containedHostPath(root: string, candidate: string): Promise<string> {
|
||||
const target = await realpath(candidate);
|
||||
if (!isAllowedDesktopPath(target, [root])) {
|
||||
throw new Error("Path escapes the project workspace");
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
@ -323,7 +323,11 @@ export function inferScript(
|
||||
{
|
||||
name: "computer_act",
|
||||
args: {
|
||||
actions: [{ kind: "type", text: typed }],
|
||||
// Click into the focused field first so the visible cursor has a target.
|
||||
actions: [
|
||||
{ kind: "click", x: 640, y: 400 },
|
||||
{ kind: "type", text: typed },
|
||||
],
|
||||
observe: true,
|
||||
},
|
||||
},
|
||||
@ -450,6 +454,20 @@ export function inferScript(
|
||||
},
|
||||
];
|
||||
}
|
||||
if (lower.includes("present") && lower.includes("report")) {
|
||||
const filePath =
|
||||
/(?:called|named|path|file)\s+([A-Za-z0-9._/-]+)/i.exec(prompt)?.[1] ?? "report.md";
|
||||
const content =
|
||||
"# Report\n\n| Item | Status |\n| --- | --- |\n| Research | Done |\n| Draft | Done |\n";
|
||||
return [
|
||||
{ assistant: "writing the report and putting it next to our chat." },
|
||||
{ toolCalls: [{ name: "write_file", args: { path: filePath, content } }] },
|
||||
{
|
||||
toolCalls: [{ name: "attach_file", args: { path: filePath, present: true } }],
|
||||
complete: true,
|
||||
},
|
||||
];
|
||||
}
|
||||
if (lower.includes("attach") && (lower.includes("thread") || lower.includes("into the thread"))) {
|
||||
const said = /says?\s+(.+)$/i.exec(prompt)?.[1]?.replace(/[.]+$/, "") ?? prompt;
|
||||
const content = `${said.trim()}\n`;
|
||||
|
||||
@ -46,6 +46,39 @@ describe("current-turn thread files", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("marks a file the bot presents as its result", async () => {
|
||||
const deps = {
|
||||
prisma: {
|
||||
artifact: {
|
||||
create: vi.fn(
|
||||
async ({ data }: { data: { name: string; mimeType: string; size: number } }) => ({
|
||||
id: "artifact-1",
|
||||
...data,
|
||||
}),
|
||||
),
|
||||
},
|
||||
} as unknown as PrismaClient,
|
||||
artifacts: {
|
||||
put: vi.fn().mockResolvedValue({ id: "stored-1", hash: "hash" }),
|
||||
remove: vi.fn(),
|
||||
} as unknown as ArtifactStore,
|
||||
};
|
||||
const input = {
|
||||
spaceId: "workspace-1",
|
||||
userId: "user-1",
|
||||
botId: "bot-1",
|
||||
runId: "run-1",
|
||||
filePath: "out/report.md",
|
||||
bytes: new TextEncoder().encode("# Report\n"),
|
||||
operationId: "attach-1",
|
||||
};
|
||||
|
||||
const presented = await attachWorkspaceFileToThread(deps, { ...input, present: true });
|
||||
expect(presented.block).toMatchObject({ kind: "file", name: "report.md", present: true });
|
||||
const attached = await attachWorkspaceFileToThread(deps, input);
|
||||
expect(attached.block).not.toHaveProperty("present");
|
||||
});
|
||||
|
||||
it("copies non-image attachments into the bot workspace and describes their paths", async () => {
|
||||
const findMany = vi.fn().mockResolvedValue([
|
||||
{
|
||||
|
||||
@ -38,6 +38,7 @@ export async function attachWorkspaceFileToThread(
|
||||
filePath: string;
|
||||
bytes: Uint8Array;
|
||||
operationId: string;
|
||||
present?: boolean;
|
||||
},
|
||||
): Promise<{ artifactId: string; block: Extract<MessageBlock, { kind: "image" | "file" }> }> {
|
||||
const name = path.basename(input.filePath) || input.filePath;
|
||||
@ -79,15 +80,13 @@ export async function attachWorkspaceFileToThread(
|
||||
await deps.artifacts.remove(stored.id, context).catch(() => undefined);
|
||||
throw error;
|
||||
});
|
||||
return {
|
||||
artifactId: row.id,
|
||||
block: messageBlockForArtifact({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
mimeType: row.mimeType,
|
||||
size: row.size,
|
||||
}),
|
||||
};
|
||||
const block = messageBlockForArtifact({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
mimeType: row.mimeType,
|
||||
size: row.size,
|
||||
});
|
||||
return { artifactId: row.id, block: input.present ? { ...block, present: true } : block };
|
||||
}
|
||||
|
||||
export async function materializeCurrentTurnFiles(
|
||||
|
||||
38
packages/contracts/src/computer-pointer.ts
Normal file
38
packages/contracts/src/computer-pointer.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import * as z from "zod";
|
||||
|
||||
export const ComputerPointerAction = z.enum([
|
||||
"move",
|
||||
"click",
|
||||
"double_click",
|
||||
"right_click",
|
||||
"down",
|
||||
"up",
|
||||
"type",
|
||||
"key",
|
||||
"scroll",
|
||||
]);
|
||||
export type ComputerPointerAction = z.infer<typeof ComputerPointerAction>;
|
||||
|
||||
/** Screen pixels; absent for keyboard steps that act wherever the pointer already is. */
|
||||
export const ComputerPointerStep = z.object({
|
||||
action: ComputerPointerAction,
|
||||
x: z.number().int().min(0).max(100_000).optional(),
|
||||
y: z.number().int().min(0).max(100_000).optional(),
|
||||
});
|
||||
export type ComputerPointerStep = z.infer<typeof ComputerPointerStep>;
|
||||
|
||||
export const MAX_COMPUTER_POINTER_STEPS = 48;
|
||||
|
||||
/**
|
||||
* Payload of a `thread.computer` event with `kind: "pointer"`: where a bot is about to point,
|
||||
* click or type on its computer screen, so clients can draw its cursor over the live screen.
|
||||
*/
|
||||
export const ComputerPointerPayload = z.object({
|
||||
kind: z.literal("pointer"),
|
||||
screen: z.object({
|
||||
width: z.number().int().positive().max(100_000),
|
||||
height: z.number().int().positive().max(100_000),
|
||||
}),
|
||||
steps: z.array(ComputerPointerStep).min(1).max(MAX_COMPUTER_POINTER_STEPS),
|
||||
});
|
||||
export type ComputerPointerPayload = z.infer<typeof ComputerPointerPayload>;
|
||||
@ -38,20 +38,39 @@ export interface RakazoDesktopOAuthCallback {
|
||||
|
||||
export interface RakazoDesktopBrowserPreviewState {
|
||||
url: string;
|
||||
devServerRunning: boolean;
|
||||
title: string;
|
||||
loading: boolean;
|
||||
canGoBack: boolean;
|
||||
canGoForward: boolean;
|
||||
/** The last main-frame load failure for `url`, cleared by the next successful load. */
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** Optional desktop bridge for the in-app browser preview panel. Absent in the browser. */
|
||||
/** Rectangle in CSS pixels of the app window's content area. */
|
||||
export interface RakazoDesktopBrowserBounds {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional desktop bridge for the in-app browser: a native Chromium view the main process lays
|
||||
* over the panel the renderer reserves for it. Absent in the browser, where an iframe is used.
|
||||
*/
|
||||
export interface RakazoDesktopBrowserPreview {
|
||||
getState: () => Promise<RakazoDesktopBrowserPreviewState>;
|
||||
/** Shows the view over these bounds; call again whenever the reserved area moves. */
|
||||
show: (bounds: RakazoDesktopBrowserBounds) => Promise<void>;
|
||||
hide: () => Promise<void>;
|
||||
/** Only http(s) URLs load; anything else is rejected. */
|
||||
navigate: (url: string) => Promise<void>;
|
||||
back: () => Promise<void>;
|
||||
forward: () => Promise<void>;
|
||||
reload: () => Promise<void>;
|
||||
onNavigate: (listener: (url: string) => void) => () => void;
|
||||
onLoadingProgress: (listener: (progress: number) => void) => () => void;
|
||||
devServer: {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
};
|
||||
stop: () => Promise<void>;
|
||||
openDevTools: () => Promise<void>;
|
||||
state: () => Promise<RakazoDesktopBrowserPreviewState>;
|
||||
onState: (listener: (state: RakazoDesktopBrowserPreviewState) => void) => () => void;
|
||||
}
|
||||
|
||||
export interface RakazoDesktop {
|
||||
|
||||
@ -240,6 +240,8 @@ export const MessageBlock = z.discriminatedUnion("kind", [
|
||||
artifactId: Id,
|
||||
mimeType: z.string(),
|
||||
name: z.string(),
|
||||
/** The bot presented this as its result; clients may show it beside the conversation. */
|
||||
present: z.boolean().optional(),
|
||||
}),
|
||||
z.object({
|
||||
kind: z.literal("file"),
|
||||
@ -247,6 +249,8 @@ export const MessageBlock = z.discriminatedUnion("kind", [
|
||||
mimeType: z.string(),
|
||||
name: z.string(),
|
||||
size: z.number().int().nonnegative(),
|
||||
/** The bot presented this as its result; clients may show it beside the conversation. */
|
||||
present: z.boolean().optional(),
|
||||
}),
|
||||
z.object({
|
||||
kind: z.literal("handoff"),
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
export * from "./ai-consent.js";
|
||||
export * from "./attachments.js";
|
||||
export * from "./bot-secrets.js";
|
||||
export * from "./computer-pointer.js";
|
||||
export * from "./desktop.js";
|
||||
export * from "./domain.js";
|
||||
export * from "./events.js";
|
||||
|
||||
@ -1,19 +1,10 @@
|
||||
// Server-only. Kept out of the package index so browser bundles never load `node:child_process`.
|
||||
import { spawn } from "node:child_process";
|
||||
import { access } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { gitStatusArgv, gitUntrackedFilesArgv, parseGitNameOnly } from "./compose-update.js";
|
||||
import { DEFAULT_UPDATE_REMOTE, parseGitStatusPorcelain } from "./self-update.js";
|
||||
|
||||
// Lazy import to avoid bundling `node:child_process` in browser bundles.
|
||||
// `defaultGitRunner` is server-only; lazy-loading ensures tree-shaking works.
|
||||
type SpawnFn = typeof import("node:child_process").spawn;
|
||||
let _spawn: SpawnFn | undefined;
|
||||
const getSpawn = (): SpawnFn => {
|
||||
if (_spawn === undefined) {
|
||||
_spawn = require("node:child_process").spawn as SpawnFn;
|
||||
}
|
||||
return _spawn;
|
||||
};
|
||||
|
||||
export interface GitCheckoutStatus {
|
||||
present: boolean;
|
||||
remoteUrl: string | null;
|
||||
@ -32,7 +23,6 @@ export type GitRunner = (args: string[], cwd: string) => Promise<GitRunnerResult
|
||||
|
||||
export async function defaultGitRunner(args: string[], cwd: string): Promise<GitRunnerResult> {
|
||||
return new Promise((resolve) => {
|
||||
const spawn = getSpawn();
|
||||
const child = spawn("git", args, { cwd, env: process.env });
|
||||
let output = "";
|
||||
child.stdout.on("data", (chunk: Buffer | string) => {
|
||||
|
||||
@ -48,3 +48,4 @@ export * from "./tool-activity.js";
|
||||
export * from "./trading-bot-status.js";
|
||||
export * from "./voice-chat-command.js";
|
||||
export * from "./voice-labels.js";
|
||||
export * from "./workspace-focus.js";
|
||||
|
||||
@ -3,6 +3,7 @@ import {
|
||||
computerInputForDomKey,
|
||||
computerScreenSize,
|
||||
mapTeachPointer,
|
||||
screenPointInViewer,
|
||||
teachCaptureKey,
|
||||
} from "./teach-recording.js";
|
||||
|
||||
@ -88,3 +89,21 @@ describe("mapTeachPointer", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenPointInViewer", () => {
|
||||
it("projects through the same letterbox the viewer draws", () => {
|
||||
const screen = { width: 1280, height: 800 };
|
||||
// A wide viewer pillarboxes the 16:10 screen.
|
||||
expect(screenPointInViewer({ x: 0, y: 0 }, { width: 1000, height: 400 }, screen)).toEqual({
|
||||
x: 180,
|
||||
y: 0,
|
||||
});
|
||||
expect(screenPointInViewer({ x: 1280, y: 800 }, { width: 1000, height: 400 }, screen)).toEqual({
|
||||
x: 820,
|
||||
y: 400,
|
||||
});
|
||||
const rect = { left: 0, top: 0, width: 640, height: 600 };
|
||||
const projected = screenPointInViewer({ x: 640, y: 400 }, rect, screen);
|
||||
expect(mapTeachPointer(projected.x, projected.y, rect, screen)).toEqual({ x: 640, y: 400 });
|
||||
});
|
||||
});
|
||||
|
||||
@ -91,3 +91,18 @@ export function mapTeachPointer(
|
||||
y: clamp(Math.round((clientY - rect.top - offsetY) / scale), screen.height - 1),
|
||||
};
|
||||
}
|
||||
|
||||
/** Inverse of {@link mapTeachPointer}: where a screen pixel lands inside the letterboxed viewer. */
|
||||
export function screenPointInViewer(
|
||||
point: { x: number; y: number },
|
||||
viewer: { width: number; height: number },
|
||||
screen: { width: number; height: number },
|
||||
): { x: number; y: number } {
|
||||
const scale = Math.min(viewer.width / screen.width, viewer.height / screen.height) || 1;
|
||||
const offsetX = (viewer.width - screen.width * scale) / 2;
|
||||
const offsetY = (viewer.height - screen.height * scale) / 2;
|
||||
return {
|
||||
x: offsetX + clamp(point.x, screen.width) * scale,
|
||||
y: offsetY + clamp(point.y, screen.height) * scale,
|
||||
};
|
||||
}
|
||||
|
||||
305
packages/core/src/workspace-focus.test.ts
Normal file
305
packages/core/src/workspace-focus.test.ts
Normal file
@ -0,0 +1,305 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
applyWorkspaceFocus,
|
||||
deriveWorkspaceFocus,
|
||||
isPaneArtifactMimeType,
|
||||
liveScreenRunId,
|
||||
parseScreenPaneHabit,
|
||||
recordScreenPaneChoice,
|
||||
screenPaneMode,
|
||||
type WorkspaceFocusSnapshot,
|
||||
} from "./workspace-focus.js";
|
||||
|
||||
const runningComputer = {
|
||||
kind: "docker",
|
||||
state: "running",
|
||||
screenAvailable: true,
|
||||
takeoverRequested: false,
|
||||
};
|
||||
|
||||
function snapshot(overrides: Partial<WorkspaceFocusSnapshot> = {}): WorkspaceFocusSnapshot {
|
||||
return {
|
||||
messages: [],
|
||||
run: { id: "run-1", status: "running" },
|
||||
computer: runningComputer,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const report = {
|
||||
id: "msg-report",
|
||||
role: "bot",
|
||||
runId: "run-1",
|
||||
blocks: [
|
||||
{
|
||||
kind: "file",
|
||||
artifactId: "art-1",
|
||||
name: "report.md",
|
||||
mimeType: "text/markdown",
|
||||
present: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("deriveWorkspaceFocus", () => {
|
||||
it("keeps a plain conversation when the bot only chats", () => {
|
||||
const focus = deriveWorkspaceFocus(
|
||||
snapshot({ messages: [{ id: "m", runId: "run-1", blocks: [{ kind: "text" }] }] }),
|
||||
);
|
||||
expect(focus.pane).toBeNull();
|
||||
expect(focus.key).toBe("none|ask:none");
|
||||
});
|
||||
|
||||
it("shows the screen once the working run acts on it", () => {
|
||||
expect(deriveWorkspaceFocus(snapshot(), { screenRunId: "run-1" }).pane).toBe("screen");
|
||||
expect(deriveWorkspaceFocus(snapshot(), { screenRunId: "run-old" }).pane).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps showing the screen while the bot pointed at it moments ago", () => {
|
||||
const finished = snapshot({ run: { id: "run-1", status: "completed" } });
|
||||
expect(deriveWorkspaceFocus(finished, { screenActive: true }).pane).toBe("screen");
|
||||
expect(
|
||||
deriveWorkspaceFocus(finished, { screenActive: true, screenMode: "never" }).pane,
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("returns to the conversation when the run finishes", () => {
|
||||
const focus = deriveWorkspaceFocus(snapshot({ run: { id: "run-1", status: "completed" } }), {
|
||||
screenRunId: "run-1",
|
||||
});
|
||||
expect(focus.pane).toBeNull();
|
||||
});
|
||||
|
||||
it("never offers a screen it cannot embed", () => {
|
||||
for (const computer of [
|
||||
{ ...runningComputer, kind: "desktop" },
|
||||
{ ...runningComputer, kind: "none" },
|
||||
null,
|
||||
]) {
|
||||
expect(
|
||||
deriveWorkspaceFocus(snapshot({ computer }), { screenRunId: "run-1" }).pane,
|
||||
).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("trusts screen work over a computer status that has not caught up with the boot", () => {
|
||||
const stale = { ...runningComputer, state: "stopped", screenAvailable: false };
|
||||
expect(deriveWorkspaceFocus(snapshot({ computer: stale }), { screenRunId: "run-1" }).pane).toBe(
|
||||
"screen",
|
||||
);
|
||||
expect(
|
||||
deriveWorkspaceFocus(
|
||||
snapshot({
|
||||
run: { id: "run-1", status: "waiting_takeover" },
|
||||
computer: { ...stale, takeoverRequested: true },
|
||||
}),
|
||||
).pane,
|
||||
).toBe("screen");
|
||||
// Without evidence of screen work, a learned default never boots a stopped computer.
|
||||
expect(
|
||||
deriveWorkspaceFocus(snapshot({ computer: stale }), { screenMode: "always" }).pane,
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("shows the screen for a takeover request even when the user turned auto off", () => {
|
||||
const focus = deriveWorkspaceFocus(
|
||||
snapshot({
|
||||
run: { id: "run-1", status: "waiting_takeover" },
|
||||
computer: { ...runningComputer, takeoverRequested: true },
|
||||
}),
|
||||
{ screenMode: "never" },
|
||||
);
|
||||
expect(focus).toMatchObject({ pane: "screen", takeover: true });
|
||||
});
|
||||
|
||||
it("follows the learned screen mode", () => {
|
||||
expect(deriveWorkspaceFocus(snapshot(), { screenMode: "always" }).pane).toBe("screen");
|
||||
expect(
|
||||
deriveWorkspaceFocus(snapshot(), { screenMode: "never", screenRunId: "run-1" }).pane,
|
||||
).toBeNull();
|
||||
expect(
|
||||
deriveWorkspaceFocus(snapshot({ run: { id: "run-1", status: "completed" } }), {
|
||||
screenMode: "always",
|
||||
}).pane,
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("shows the file the bot presented as its latest result", () => {
|
||||
const focus = deriveWorkspaceFocus(
|
||||
snapshot({ run: { id: "run-1", status: "completed" }, messages: [report] }),
|
||||
);
|
||||
expect(focus.pane).toBe("artifact");
|
||||
expect(focus.artifact).toEqual({
|
||||
messageId: "msg-report",
|
||||
artifactId: "art-1",
|
||||
name: "report.md",
|
||||
mimeType: "text/markdown",
|
||||
});
|
||||
expect(focus.key).toBe("artifact:art-1|ask:none");
|
||||
});
|
||||
|
||||
it("ignores files that were only attached, presented before the user wrote again, or not previewable", () => {
|
||||
const attached = {
|
||||
...report,
|
||||
blocks: [{ ...report.blocks[0]!, present: undefined }],
|
||||
};
|
||||
expect(deriveWorkspaceFocus(snapshot({ messages: [attached] })).pane).toBeNull();
|
||||
const followUp = { id: "msg-user", role: "user", runId: "run-2", blocks: [{ kind: "text" }] };
|
||||
expect(deriveWorkspaceFocus(snapshot({ messages: [report, followUp] })).pane).toBeNull();
|
||||
// Another member's newer run in a group does not hide the result.
|
||||
expect(
|
||||
deriveWorkspaceFocus(
|
||||
snapshot({ run: { id: "run-9", status: "running" }, messages: [report] }),
|
||||
).pane,
|
||||
).toBe("artifact");
|
||||
const pdf = {
|
||||
...report,
|
||||
blocks: [{ ...report.blocks[0]!, mimeType: "application/pdf", name: "report.pdf" }],
|
||||
};
|
||||
expect(deriveWorkspaceFocus(snapshot({ messages: [pdf] })).pane).toBeNull();
|
||||
});
|
||||
|
||||
it("prefers live screen work over a presented file", () => {
|
||||
expect(
|
||||
deriveWorkspaceFocus(snapshot({ messages: [report] }), { screenRunId: "run-1" }).pane,
|
||||
).toBe("screen");
|
||||
});
|
||||
|
||||
it("names the ask the bot is waiting on", () => {
|
||||
const focus = deriveWorkspaceFocus(
|
||||
snapshot({
|
||||
run: { id: "run-1", status: "waiting_input" },
|
||||
messages: [{ id: "ask-1", runId: "run-1", blocks: [{ kind: "ask", status: "pending" }] }],
|
||||
}),
|
||||
);
|
||||
expect(focus.needsYouMessageId).toBe("ask-1");
|
||||
expect(focus.key).toBe("none|ask:ask-1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyWorkspaceFocus", () => {
|
||||
const plain = deriveWorkspaceFocus(snapshot());
|
||||
const screen = deriveWorkspaceFocus(snapshot(), { screenRunId: "run-1" });
|
||||
const artifact = deriveWorkspaceFocus(
|
||||
snapshot({ run: { id: "run-1", status: "completed" }, messages: [report] }),
|
||||
);
|
||||
const closed = { open: null, auto: false } as const;
|
||||
|
||||
it("treats the first focus of a conversation as a baseline", () => {
|
||||
expect(applyWorkspaceFocus(null, screen, closed)).toBeNull();
|
||||
});
|
||||
|
||||
it("opens the pane on an edge and closes it again when the work ends", () => {
|
||||
const opened = applyWorkspaceFocus(plain, screen, closed);
|
||||
expect(opened).toEqual({ open: "screen", auto: true });
|
||||
expect(applyWorkspaceFocus(screen, plain, opened!)).toEqual(closed);
|
||||
});
|
||||
|
||||
it("moves from the screen to the presented result", () => {
|
||||
expect(applyWorkspaceFocus(screen, artifact, { open: "screen", auto: true })).toEqual({
|
||||
open: "artifact",
|
||||
auto: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a presented file open while the user follows up", () => {
|
||||
const followUp = deriveWorkspaceFocus(
|
||||
snapshot({
|
||||
run: { id: "run-2", status: "running" },
|
||||
messages: [report, { id: "msg-user", role: "user", blocks: [{ kind: "text" }] }],
|
||||
}),
|
||||
);
|
||||
expect(followUp.pane).toBeNull();
|
||||
expect(applyWorkspaceFocus(artifact, followUp, { open: "artifact", auto: true })).toBeNull();
|
||||
});
|
||||
|
||||
it("never replaces or closes what the user opened by hand", () => {
|
||||
expect(applyWorkspaceFocus(plain, screen, { open: "other", auto: false })).toBeNull();
|
||||
expect(applyWorkspaceFocus(plain, artifact, { open: "screen", auto: false })).toBeNull();
|
||||
expect(applyWorkspaceFocus(screen, plain, { open: "screen", auto: false })).toBeNull();
|
||||
expect(applyWorkspaceFocus(plain, screen, { open: "screen", auto: false })).toBeNull();
|
||||
});
|
||||
|
||||
it("swaps in a newer presented file", () => {
|
||||
const newer = deriveWorkspaceFocus(
|
||||
snapshot({
|
||||
run: { id: "run-1", status: "completed" },
|
||||
messages: [
|
||||
report,
|
||||
{
|
||||
...report,
|
||||
id: "msg-2",
|
||||
blocks: [{ ...report.blocks[0]!, artifactId: "art-2", name: "v2.md" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(applyWorkspaceFocus(artifact, newer, { open: "artifact", auto: false })).toEqual({
|
||||
open: "artifact",
|
||||
auto: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores focus changes that only concern a new ask", () => {
|
||||
const asking = { ...plain, needsYouMessageId: "ask-1", key: "none|ask:ask-1" };
|
||||
expect(applyWorkspaceFocus(plain, asking, closed)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("screen pane habit", () => {
|
||||
it("learns to always show the screen after repeated manual opens", () => {
|
||||
let habit = 0;
|
||||
for (let index = 0; index < 3; index += 1) habit = recordScreenPaneChoice(habit, "opened");
|
||||
expect(screenPaneMode(habit)).toBe("always");
|
||||
expect(recordScreenPaneChoice(habit, "opened")).toBe(3);
|
||||
});
|
||||
|
||||
it("learns to stop opening it after repeated dismissals, and recovers on a manual open", () => {
|
||||
let habit = recordScreenPaneChoice(2, "dismissed");
|
||||
expect(screenPaneMode(habit)).toBe("auto");
|
||||
habit = recordScreenPaneChoice(habit, "dismissed");
|
||||
expect(screenPaneMode(habit)).toBe("never");
|
||||
expect(screenPaneMode(recordScreenPaneChoice(habit, "opened"))).toBe("auto");
|
||||
});
|
||||
|
||||
it("parses stored habits defensively", () => {
|
||||
expect(parseScreenPaneHabit(null)).toBe(0);
|
||||
expect(parseScreenPaneHabit("2")).toBe(2);
|
||||
expect(parseScreenPaneHabit("99")).toBe(3);
|
||||
expect(parseScreenPaneHabit("-9")).toBe(-2);
|
||||
expect(parseScreenPaneHabit("x")).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isPaneArtifactMimeType", () => {
|
||||
it("accepts text and images, not PDFs", () => {
|
||||
expect(isPaneArtifactMimeType("text/csv")).toBe(true);
|
||||
expect(isPaneArtifactMimeType("image/png")).toBe(true);
|
||||
expect(isPaneArtifactMimeType("application/pdf")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("liveScreenRunId", () => {
|
||||
const pointer = {
|
||||
type: "thread.computer",
|
||||
runId: "run-1",
|
||||
createdAt: "2026-01-01T00:00:10.000Z",
|
||||
payload: {
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [{ action: "click", x: 640, y: 400 }],
|
||||
},
|
||||
};
|
||||
const now = Date.parse("2026-01-01T00:00:20.000Z");
|
||||
|
||||
it("names the run that is pointing at the screen right now", () => {
|
||||
expect(liveScreenRunId(pointer, now)).toBe("run-1");
|
||||
});
|
||||
|
||||
it("ignores stale, malformed and unrelated events", () => {
|
||||
expect(liveScreenRunId(pointer, now + 60_000)).toBeNull();
|
||||
expect(liveScreenRunId({ ...pointer, payload: { kind: "pointer" } }, now)).toBeNull();
|
||||
expect(liveScreenRunId({ ...pointer, runId: undefined }, now)).toBeNull();
|
||||
expect(liveScreenRunId({ ...pointer, type: "thread.progress" }, now)).toBeNull();
|
||||
});
|
||||
});
|
||||
230
packages/core/src/workspace-focus.ts
Normal file
230
packages/core/src/workspace-focus.ts
Normal file
@ -0,0 +1,230 @@
|
||||
import { ComputerPointerPayload, isAttachmentImageMimeType } from "@rakazo/contracts";
|
||||
import { latestAnswerableAskMessageId } from "./answerable-ask.js";
|
||||
|
||||
/** What the pane beside a conversation shows while a bot works. */
|
||||
export type WorkspacePane = "screen" | "artifact";
|
||||
|
||||
/** Learned per bot: whether the live screen opens on its own while the bot works. */
|
||||
export type ScreenPaneMode = "auto" | "always" | "never";
|
||||
|
||||
export type WorkspaceArtifact = {
|
||||
messageId: string;
|
||||
artifactId: string;
|
||||
name: string;
|
||||
mimeType: string;
|
||||
};
|
||||
|
||||
export type WorkspaceFocus = {
|
||||
pane: WorkspacePane | null;
|
||||
artifact: WorkspaceArtifact | null;
|
||||
/** The bot asked the user to take over its screen. */
|
||||
takeover: boolean;
|
||||
/** The ask message the bot is waiting on. */
|
||||
needsYouMessageId: string | null;
|
||||
/** Changes only when the focus target changes, so clients react to edges, not every event. */
|
||||
key: string;
|
||||
};
|
||||
|
||||
type FocusBlock = {
|
||||
kind: string;
|
||||
artifactId?: string;
|
||||
name?: string;
|
||||
mimeType?: string;
|
||||
present?: boolean;
|
||||
status?: string;
|
||||
};
|
||||
|
||||
type FocusRun = { id: string; status: string };
|
||||
|
||||
export type WorkspaceFocusSnapshot = {
|
||||
messages: readonly {
|
||||
id: string;
|
||||
role?: string;
|
||||
runId?: string | null;
|
||||
blocks: readonly FocusBlock[];
|
||||
}[];
|
||||
run?: FocusRun | null;
|
||||
activeRuns?: readonly FocusRun[];
|
||||
computer?: {
|
||||
kind: string;
|
||||
state: string;
|
||||
screenAvailable: boolean;
|
||||
takeoverRequested: boolean;
|
||||
} | null;
|
||||
};
|
||||
|
||||
export type WorkspaceFocusLive = {
|
||||
/** Run that last announced pointer activity on the bot's screen. */
|
||||
screenRunId?: string | null;
|
||||
/** The bot pointed at its screen moments ago, even if that run already finished. */
|
||||
screenActive?: boolean;
|
||||
screenMode?: ScreenPaneMode;
|
||||
};
|
||||
|
||||
const WORKING_RUN_STATUSES = new Set(["leased", "running", "waiting_takeover"]);
|
||||
const PANE_TEXT_MIME_TYPES = new Set([
|
||||
"text/markdown",
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"application/json",
|
||||
]);
|
||||
|
||||
/** Older pointer announcements describe actions that already finished. */
|
||||
export const POINTER_EVENT_MAX_AGE_MS = 30_000;
|
||||
|
||||
/** The run behind a fresh pointer announcement on a bot's screen, or null for any other event. */
|
||||
export function liveScreenRunId(
|
||||
event: { type: string; runId?: string | null; createdAt?: string; payload?: unknown },
|
||||
now = Date.now(),
|
||||
): string | null {
|
||||
if (event.type !== "thread.computer" || !event.runId) return null;
|
||||
if (!ComputerPointerPayload.safeParse(event.payload).success) return null;
|
||||
const at = event.createdAt ? Date.parse(event.createdAt) : Number.NaN;
|
||||
if (Number.isFinite(at) && now - at > POINTER_EVENT_MAX_AGE_MS) return null;
|
||||
return event.runId;
|
||||
}
|
||||
|
||||
/** Files the side pane can render without a plugin; others stay download cards. */
|
||||
export function isPaneArtifactMimeType(mimeType: string): boolean {
|
||||
return PANE_TEXT_MIME_TYPES.has(mimeType) || isAttachmentImageMimeType(mimeType);
|
||||
}
|
||||
|
||||
function workingRunIds(snapshot: WorkspaceFocusSnapshot): Set<string> {
|
||||
const runs = snapshot.activeRuns ?? (snapshot.run ? [snapshot.run] : []);
|
||||
return new Set(runs.filter((run) => WORKING_RUN_STATUSES.has(run.status)).map((run) => run.id));
|
||||
}
|
||||
|
||||
/** The newest file a bot presented since the user last wrote; a new request starts fresh. */
|
||||
function presentedArtifact(snapshot: WorkspaceFocusSnapshot): WorkspaceArtifact | null {
|
||||
for (let index = snapshot.messages.length - 1; index >= 0; index -= 1) {
|
||||
const message = snapshot.messages[index];
|
||||
if (!message) continue;
|
||||
if (message.role === "user") return null;
|
||||
for (let blockIndex = message.blocks.length - 1; blockIndex >= 0; blockIndex -= 1) {
|
||||
const block = message.blocks[blockIndex];
|
||||
if (
|
||||
(block?.kind === "file" || block?.kind === "image") &&
|
||||
block.present &&
|
||||
block.artifactId &&
|
||||
block.mimeType &&
|
||||
isPaneArtifactMimeType(block.mimeType)
|
||||
) {
|
||||
return {
|
||||
messageId: message.id,
|
||||
artifactId: block.artifactId,
|
||||
name: block.name ?? "",
|
||||
mimeType: block.mimeType,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives the pane a conversation should show from what the bot is actually doing: a
|
||||
* takeover request or live work on its screen shows the screen, a file the bot presented
|
||||
* as its latest result shows that file, and everything else stays a plain conversation.
|
||||
*/
|
||||
export function deriveWorkspaceFocus(
|
||||
snapshot: WorkspaceFocusSnapshot | null,
|
||||
live: WorkspaceFocusLive = {},
|
||||
): WorkspaceFocus {
|
||||
if (!snapshot) {
|
||||
return { pane: null, artifact: null, takeover: false, needsYouMessageId: null, key: "none" };
|
||||
}
|
||||
const needsYouMessageId = latestAnswerableAskMessageId({
|
||||
messages: snapshot.messages,
|
||||
run: snapshot.run,
|
||||
activeRuns: snapshot.activeRuns,
|
||||
});
|
||||
const computer = snapshot.computer;
|
||||
const working = workingRunIds(snapshot);
|
||||
const mode = live.screenMode ?? "auto";
|
||||
const pointerWork =
|
||||
Boolean(live.screenActive) || (live.screenRunId != null && working.has(live.screenRunId));
|
||||
// A desktop host or a bot without a computer has no screen to embed.
|
||||
const embeddable = Boolean(computer) && computer?.kind !== "desktop" && computer?.kind !== "none";
|
||||
// A bot that points at its screen or asks for a takeover has a running screen, even when the
|
||||
// client's computer status has not caught up with a boot that happened inside the run.
|
||||
const takeover = embeddable && Boolean(computer?.takeoverRequested);
|
||||
const screenWork =
|
||||
embeddable &&
|
||||
mode !== "never" &&
|
||||
(pointerWork || (mode === "always" && working.size > 0 && Boolean(computer?.screenAvailable)));
|
||||
const artifact = presentedArtifact(snapshot);
|
||||
const pane: WorkspacePane | null =
|
||||
takeover || screenWork ? "screen" : artifact ? "artifact" : null;
|
||||
const target =
|
||||
pane === "screen"
|
||||
? `screen:${takeover ? "takeover" : "work"}`
|
||||
: pane === "artifact" && artifact
|
||||
? `artifact:${artifact.artifactId}`
|
||||
: "none";
|
||||
return {
|
||||
pane,
|
||||
artifact,
|
||||
takeover,
|
||||
needsYouMessageId,
|
||||
key: `${target}|ask:${needsYouMessageId ?? "none"}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** What occupies the pane right now; "other" is anything the user opened that is not a focus pane. */
|
||||
export type PaneState = { open: WorkspacePane | "other" | null; auto: boolean };
|
||||
|
||||
/**
|
||||
* Moves the pane only on focus edges. It never replaces something the user opened by hand,
|
||||
* and it only closes a screen it opened itself; a presented file stays until the user closes it
|
||||
* or the bot presents another.
|
||||
*/
|
||||
export function applyWorkspaceFocus(
|
||||
previous: WorkspaceFocus | null,
|
||||
next: WorkspaceFocus,
|
||||
pane: PaneState,
|
||||
): PaneState | null {
|
||||
if (!previous) return null;
|
||||
const sameTarget =
|
||||
previous.pane === next.pane &&
|
||||
(next.pane !== "artifact" || previous.artifact?.artifactId === next.artifact?.artifactId);
|
||||
if (sameTarget) return null;
|
||||
if (next.pane) {
|
||||
// A newer presented file replaces the one on view; the screen is already showing.
|
||||
if (pane.open === next.pane) return next.pane === "artifact" ? pane : null;
|
||||
if (pane.open && !pane.auto) return null;
|
||||
return { open: next.pane, auto: true };
|
||||
}
|
||||
if (pane.auto && pane.open === "screen" && previous.pane === "screen") {
|
||||
return { open: null, auto: false };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Signed habit score: positive when the user keeps opening the screen, negative when they close it. */
|
||||
export type ScreenPaneHabit = number;
|
||||
|
||||
const ALWAYS_AT = 3;
|
||||
const NEVER_AT = -2;
|
||||
|
||||
export function screenPaneMode(habit: ScreenPaneHabit): ScreenPaneMode {
|
||||
if (habit >= ALWAYS_AT) return "always";
|
||||
if (habit <= NEVER_AT) return "never";
|
||||
return "auto";
|
||||
}
|
||||
|
||||
/**
|
||||
* Learns from what the user does while a bot works: opening the screen by hand moves toward
|
||||
* showing it every time, closing an automatically opened screen moves toward never opening it.
|
||||
*/
|
||||
export function recordScreenPaneChoice(
|
||||
habit: ScreenPaneHabit,
|
||||
choice: "opened" | "dismissed",
|
||||
): ScreenPaneHabit {
|
||||
if (choice === "opened") return Math.min(ALWAYS_AT, Math.max(habit, 0) + 1);
|
||||
return Math.max(NEVER_AT, Math.min(habit, 0) - 1);
|
||||
}
|
||||
|
||||
export function parseScreenPaneHabit(value: string | null | undefined): ScreenPaneHabit {
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) ? Math.max(NEVER_AT, Math.min(ALWAYS_AT, parsed)) : 0;
|
||||
}
|
||||
@ -1,12 +1,12 @@
|
||||
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { PrismaClient } from "./client.js";
|
||||
import {
|
||||
assertProjectWorkspaceDirectory,
|
||||
InvalidProjectWorkspacePathError,
|
||||
normalizeProjectWorkspacePath,
|
||||
resolveProjectWorkspaceDirectory,
|
||||
wireProjectWorkspace,
|
||||
} from "./project-workspace.js";
|
||||
|
||||
@ -43,11 +43,21 @@ describe("project workspace wiring", () => {
|
||||
|
||||
it("rejects missing folders", async () => {
|
||||
const missing = path.join(tmpdir(), `missing-${Date.now()}`);
|
||||
await expect(assertProjectWorkspaceDirectory(missing)).rejects.toBeInstanceOf(
|
||||
await expect(resolveProjectWorkspaceDirectory(missing)).rejects.toBeInstanceOf(
|
||||
InvalidProjectWorkspacePathError,
|
||||
);
|
||||
});
|
||||
|
||||
it("resolves a symlinked folder to the real folder behind it", async () => {
|
||||
const root = tempDir("project-workspace-symlink-");
|
||||
const real = path.join(root, "real");
|
||||
mkdirSync(real);
|
||||
symlinkSync(real, path.join(root, "link"));
|
||||
expect(await resolveProjectWorkspaceDirectory(path.join(root, "link"))).toBe(
|
||||
realpathSync(real),
|
||||
);
|
||||
});
|
||||
|
||||
it("persists workspace and git metadata on the space", async () => {
|
||||
const root = tempDir("project-workspace-wire-");
|
||||
mkdirSync(path.join(root, ".git"));
|
||||
@ -62,9 +72,9 @@ describe("project workspace wiring", () => {
|
||||
|
||||
const wired = await wireProjectWorkspace(prisma as unknown as Pick<PrismaClient, "space">, {
|
||||
spaceId: "space-1",
|
||||
workspacePath: root,
|
||||
workspacePath: await resolveProjectWorkspaceDirectory(root),
|
||||
});
|
||||
expect(wired.path).toBe(path.resolve(root));
|
||||
expect(wired.path).toBe(realpathSync(root));
|
||||
expect(wired.git).toEqual({
|
||||
remoteUrl: "https://github.com/example/demo.git",
|
||||
defaultBranch: "main",
|
||||
@ -73,7 +83,7 @@ describe("project workspace wiring", () => {
|
||||
dirtyPaths: [],
|
||||
});
|
||||
expect(updates[0]).toMatchObject({
|
||||
workspacePath: path.resolve(root),
|
||||
workspacePath: realpathSync(root),
|
||||
gitRemoteUrl: "https://github.com/example/demo.git",
|
||||
gitDefaultBranch: "main",
|
||||
gitHeadCommit: "abc123",
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { access, stat } from "node:fs/promises";
|
||||
import { realpath, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { readGitCheckout } from "@rakazo/core/git-checkout";
|
||||
import type { PrismaClient } from "./client.js";
|
||||
@ -22,11 +22,13 @@ export function normalizeProjectWorkspacePath(input: string): string {
|
||||
return path.resolve(trimmed);
|
||||
}
|
||||
|
||||
export async function assertProjectWorkspaceDirectory(workspacePath: string): Promise<void> {
|
||||
const resolved = normalizeProjectWorkspacePath(workspacePath);
|
||||
/** The real folder behind a path, so a symlink cannot later point the link somewhere else. */
|
||||
export async function resolveProjectWorkspaceDirectory(workspacePath: string): Promise<string> {
|
||||
const normalized = normalizeProjectWorkspacePath(workspacePath);
|
||||
let resolved: string;
|
||||
let info: Awaited<ReturnType<typeof stat>>;
|
||||
try {
|
||||
await access(resolved);
|
||||
resolved = await realpath(normalized);
|
||||
info = await stat(resolved);
|
||||
} catch {
|
||||
throw new InvalidProjectWorkspacePathError("That folder does not exist or is not readable.");
|
||||
@ -34,6 +36,7 @@ export async function assertProjectWorkspaceDirectory(workspacePath: string): Pr
|
||||
if (!info.isDirectory()) {
|
||||
throw new InvalidProjectWorkspacePathError("That path is not a folder.");
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
export interface WiredProjectWorkspace {
|
||||
@ -47,19 +50,17 @@ export interface WiredProjectWorkspace {
|
||||
} | null;
|
||||
}
|
||||
|
||||
/** Persist a host folder and optional git metadata on a newly created space. */
|
||||
/** Persist a folder from `resolveProjectWorkspaceDirectory`, and its git metadata, on a new space. */
|
||||
export async function wireProjectWorkspace(
|
||||
prisma: Pick<PrismaClient, "space">,
|
||||
input: { spaceId: string; workspacePath: string },
|
||||
): Promise<WiredProjectWorkspace> {
|
||||
const resolved = normalizeProjectWorkspacePath(input.workspacePath);
|
||||
await assertProjectWorkspaceDirectory(resolved);
|
||||
const git = await readGitCheckout(resolved);
|
||||
const git = await readGitCheckout(input.workspacePath);
|
||||
|
||||
await prisma.space.update({
|
||||
where: { id: input.spaceId },
|
||||
data: {
|
||||
workspacePath: resolved,
|
||||
workspacePath: input.workspacePath,
|
||||
gitRemoteUrl: git.present && git.remoteUrl ? git.remoteUrl : null,
|
||||
gitDefaultBranch: git.present && git.defaultBranch ? git.defaultBranch : null,
|
||||
gitHeadCommit: git.present ? git.headCommit : null,
|
||||
@ -67,7 +68,7 @@ export async function wireProjectWorkspace(
|
||||
});
|
||||
|
||||
return {
|
||||
path: resolved,
|
||||
path: input.workspacePath,
|
||||
git:
|
||||
git.present && git.remoteUrl && git.defaultBranch
|
||||
? {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { ComposioEmulator, LocalAgentHomeStore, teamBotWorkspaceDirectory } from "@rakazo/adapters";
|
||||
@ -2185,6 +2185,110 @@ describeWithDatabase("API authorization and resource isolation", () => {
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("lets only the deployment owner link a host folder and read its files", async () => {
|
||||
const owner = await signup(app, `folder-owner-${stamp}@rakazo.test`, "Folder Owner");
|
||||
const member = await signup(app, `folder-member-${stamp}@rakazo.test`, "Folder Member");
|
||||
const ownerActor = await rpc<Actor>(app, owner, "me");
|
||||
const settings = await handles.prisma.deploymentSettings.findUniqueOrThrow({
|
||||
where: { id: "default" },
|
||||
select: { ownerUserId: true },
|
||||
});
|
||||
await handles.prisma.deploymentSettings.update({
|
||||
where: { id: "default" },
|
||||
data: { ownerUserId: ownerActor.userId },
|
||||
});
|
||||
// Stands in for any folder the API process can read, such as its own checkout.
|
||||
const hostFolder = realpathSync(mkdtempSync(path.join(tmpdir(), "rakazo-authz-host-")));
|
||||
writeFileSync(path.join(hostFolder, ".env"), "HOST_SECRET=not-a-real-secret\n");
|
||||
try {
|
||||
await expectForbidden(app, member, "spaces/create", {
|
||||
name: "Member folder",
|
||||
workspacePath: hostFolder,
|
||||
});
|
||||
const memberNavigation = await rpc<SpaceNavigation>(app, member, "spaces/list");
|
||||
expect(memberNavigation.spaces.map((space) => space.name)).not.toContain("Member folder");
|
||||
|
||||
// A space linked before only the owner could link stays closed to the member.
|
||||
const legacy = await rpc<Space>(app, member, "spaces/create", { name: "Legacy folder" });
|
||||
await handles.prisma.space.update({
|
||||
where: { id: legacy.id },
|
||||
data: { workspacePath: hostFolder },
|
||||
});
|
||||
const memberBot = await rpc<Bot>(
|
||||
app,
|
||||
member,
|
||||
"bots/create",
|
||||
botInput("Member Folder Reader"),
|
||||
legacy.id,
|
||||
);
|
||||
await expectForbidden(
|
||||
app,
|
||||
member,
|
||||
"computer/files",
|
||||
{ botId: memberBot.id, path: "project" },
|
||||
legacy.id,
|
||||
);
|
||||
await expectForbidden(
|
||||
app,
|
||||
member,
|
||||
"computer/readFile",
|
||||
{ botId: memberBot.id, path: "project/.env" },
|
||||
legacy.id,
|
||||
);
|
||||
const legacyNavigation = await rpc<SpaceNavigation>(app, member, "spaces/list");
|
||||
expect(legacyNavigation.spaces.find((space) => space.id === legacy.id)?.workspace).toBeNull();
|
||||
|
||||
// A bad folder is rejected before the project exists, so fixing it and retrying leaves one.
|
||||
for (const badPath of [path.join(hostFolder, "missing"), path.join(hostFolder, ".env")]) {
|
||||
const rejected = await raw(app, owner, "spaces/create", {
|
||||
name: "Owner folder",
|
||||
workspacePath: badPath,
|
||||
});
|
||||
expect(rejected.status, badPath).toBe(400);
|
||||
}
|
||||
const linked = await rpc<Space>(app, owner, "spaces/create", {
|
||||
name: "Owner folder",
|
||||
workspacePath: hostFolder,
|
||||
});
|
||||
expect(linked.workspace?.path).toBe(hostFolder);
|
||||
const ownerNavigation = await rpc<SpaceNavigation>(app, owner, "spaces/list");
|
||||
expect(
|
||||
ownerNavigation.spaces
|
||||
.filter((space) => space.name === "Owner folder")
|
||||
.map((space) => space.workspace?.path),
|
||||
).toEqual([hostFolder]);
|
||||
const ownerBot = await rpc<Bot>(
|
||||
app,
|
||||
owner,
|
||||
"bots/create",
|
||||
botInput("Owner Folder Reader"),
|
||||
linked.id,
|
||||
);
|
||||
const listing = await rpc<Array<{ path: string }>>(
|
||||
app,
|
||||
owner,
|
||||
"computer/files",
|
||||
{ botId: ownerBot.id, path: "project" },
|
||||
linked.id,
|
||||
);
|
||||
expect(listing.map((entry) => entry.path)).toContain("project/.env");
|
||||
const file = await rpc<{ content: string }>(
|
||||
app,
|
||||
owner,
|
||||
"computer/readFile",
|
||||
{ botId: ownerBot.id, path: "project/.env" },
|
||||
linked.id,
|
||||
);
|
||||
expect(file.content).toContain("HOST_SECRET");
|
||||
} finally {
|
||||
await handles.prisma.deploymentSettings.update({
|
||||
where: { id: "default" },
|
||||
data: { ownerUserId: settings.ownerUserId },
|
||||
});
|
||||
rmSync(hostFolder, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
function botInput(name: string) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user