diff --git a/.env.example b/.env.example index 5d588c8..8ebb743 100644 --- a/.env.example +++ b/.env.example @@ -25,9 +25,9 @@ PUBLIC_POSTHOG_KEY= PUBLIC_POSTHOG_HOST=https://us.i.posthog.com SIGNUPS_ENABLED=true SIGNUP_ALLOWLIST= -# First-run discovery. Leave unset and the onboarding scan never touches the host filesystem. -# Set a home-style directory (its Projects, Documents, src, ... subfolders are scanned) to let the -# deployment owner's own onboarding discover local projects. Other accounts never scan the host. +# First-run discovery. Leave unset and onboarding never looks at the host filesystem. +# Set a folder (for example a home folder) and the deployment owner's onboarding offers to list the +# git projects in it, reading file and folder names only. Other accounts never see the step. # ONBOARDING_SCAN_HOME= ENCRYPTION_KEY=replace-with-64-random-hex-characters DATA_DIR=./data diff --git a/apps/api/src/connected-app-facts.ts b/apps/api/src/connected-app-facts.ts new file mode 100644 index 0000000..5b83abe --- /dev/null +++ b/apps/api/src/connected-app-facts.ts @@ -0,0 +1,50 @@ +import { FEATURED_CONNECTOR_LABELS, matchFeaturedConnectorId } from "@rakazo/core"; +import type { PrismaClient } from "@rakazo/db"; +import { updateUserProfile } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; + +/** + * The app's own name: a featured app's label, otherwise the label of the user's first account for + * it (later accounts may be named "Gmail 2" or renamed). + */ +async function connectedAppName( + prisma: PrismaClient, + connection: { userId: string; connectorId: string; provider: string }, +): Promise { + const featured = matchFeaturedConnectorId(connection.provider); + if (featured) return FEATURED_CONNECTOR_LABELS[featured]; + const first = await prisma.connection.findFirst({ + where: { + userId: connection.userId, + connectorId: connection.connectorId, + provider: connection.provider, + }, + orderBy: [{ createdAt: "asc" }, { id: "asc" }], + select: { displayName: true }, + }); + const name = first?.displayName.trim().slice(0, 80); + return name || null; +} + +/** + * Remembers "Uses " in the user's profile when one of their connections becomes connected. + * Revoking the connection keeps the fact; the user removes it in About you. Best effort: the + * connection itself has already succeeded, so a full profile or a database error only logs. + */ +export async function recordConnectedApp( + prisma: PrismaClient, + connection: { userId: string; connectorId: string; provider: string }, +): Promise { + try { + const name = await connectedAppName(prisma, connection); + if (!name) return; + await updateUserProfile(prisma, connection.userId, { + addFacts: [{ category: "tools", value: `Uses ${name}`, source: "connector" }], + }); + } catch (error) { + getLogger().error("connected app fact was not recorded", error, { + connectorId: connection.connectorId, + provider: connection.provider, + }); + } +} diff --git a/apps/api/src/onboarding-scan.test.ts b/apps/api/src/onboarding-scan.test.ts index ea2e625..9cbafd9 100644 --- a/apps/api/src/onboarding-scan.test.ts +++ b/apps/api/src/onboarding-scan.test.ts @@ -1,64 +1,192 @@ -import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, realpath, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { getScanStatus, startOnboardingScan } from "./onboarding-scan.js"; +import path from "node:path"; +import type { PrismaClient } from "@rakazo/db"; +import { afterAll, describe, expect, it, vi } from "vitest"; +import { + applyOnboardingScan, + discoverOnboardingScan, + onboardingScanFolder, + onboardingScanStatus, + startOnboardingScan, +} from "./onboarding-scan.js"; -const owner = { userId: "user-owner", isDeploymentOwner: true }; -const member = { userId: "user-member", isDeploymentOwner: false }; +const owner = { userId: "user-owner", spaceId: "space-owner", isDeploymentOwner: true }; +const member = { userId: "user-member", spaceId: "space-member", isDeploymentOwner: false }; +const roots: string[] = []; -async function scanHomeWithOneFile(): Promise { - const home = await mkdtemp(join(tmpdir(), "evoo-scan-")); - await mkdir(join(home, "Projects")); - await writeFile(join(home, "Projects", "notes.ts"), "export const x = 1;\n"); - return home; +/** Fails the test if the scan writes anything before its checks pass. */ +const untouchedPrisma = new Proxy( + {}, + { + get() { + throw new Error("the database must not be touched"); + }, + }, +) as PrismaClient; + +async function tempDir(): Promise { + const dir = await realpath(await mkdtemp(path.join(tmpdir(), "evoo-scan-"))); + roots.push(dir); + return dir; +} + +async function files(root: string, entries: string[]): Promise { + for (const entry of entries) { + const file = path.join(root, entry); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, "x\n"); + } +} + +async function project(root: string, relative: string, entries: string[]): Promise { + await mkdir(path.join(root, relative, ".git"), { recursive: true }); + await files(path.join(root, relative), entries); } async function completed(scanId: string, userId: string) { return await vi.waitFor( - async () => { - const status = await getScanStatus(scanId, userId); - if (status.status !== "complete" && status.status !== "failed") throw new Error("pending"); + () => { + const status = onboardingScanStatus(scanId, userId); + if (status.status === "scanning") throw new Error("pending"); return status; }, - { timeout: 10_000, interval: 50 }, + { timeout: 10_000, interval: 20 }, ); } +afterAll(async () => { + await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))); +}); + describe("onboarding scan", () => { - afterEach(() => { - vi.unstubAllEnvs(); + it("lists git projects with their languages and tools and never follows a symlink", async () => { + const home = await tempDir(); + const outside = await tempDir(); + // A home folder tracked by git is not itself a project. + await mkdir(path.join(home, ".git")); + await project(home, "code/atlas", [ + "src/a.ts", + "src/b.ts", + "src/c.ts", + "src/d.tsx", + "scripts/build.js", + "Dockerfile", + ".github/workflows/ci.yml", + "infra/main.tf", + "src/odd.constructor", + "node_modules/dep/index.js", + "node_modules/dep/more.js", + ]); + await project(home, "code/garden", ["main.py"]); + await project(home, "deep/a/b/edge", ["tool.py"]); + await project(home, "deep/a/b/c/too-deep", ["main.go"]); + await project(home, ".hidden/secret", ["main.go"]); + await project(home, "notes/node_modules/vendored", ["main.go"]); + await files(home, ["notes/todo.md"]); + await project(outside, "escaped", ["main.rs"]); + await symlink(path.join(outside, "escaped"), path.join(home, "linked")); + await symlink(outside, path.join(home, "code/atlas/src/outside")); + + const findings = await discoverOnboardingScan(home); + + expect(findings.projects).toEqual([ + { + id: "project-0", + name: "atlas", + path: "code/atlas", + languages: ["TypeScript", "JavaScript"], + tools: ["Docker", "GitHub Actions", "Terraform"], + }, + { id: "project-1", name: "garden", path: "code/garden", languages: ["Python"], tools: [] }, + { id: "project-2", name: "edge", path: "deep/a/b/edge", languages: ["Python"], tools: [] }, + ]); + expect(findings.facts).toEqual([ + { id: "language:Python", label: "Python", category: "work", value: "Codes in Python" }, + { + id: "language:JavaScript", + label: "JavaScript", + category: "work", + value: "Codes in JavaScript", + }, + { + id: "language:TypeScript", + label: "TypeScript", + category: "work", + value: "Codes in TypeScript", + }, + { id: "tool:Docker", label: "Docker", category: "tools", value: "Uses Docker" }, + { + id: "tool:GitHub Actions", + label: "GitHub Actions", + category: "tools", + value: "Uses GitHub Actions", + }, + { id: "tool:Terraform", label: "Terraform", category: "tools", value: "Uses Terraform" }, + ]); }); - it("hides a scan from every account except the one that started it", async () => { - const { scanId } = await startOnboardingScan({ actor: member }, {}); - await expect(getScanStatus(scanId, owner.userId)).rejects.toThrow(/Scan not found/); - await expect(getScanStatus(scanId, member.userId)).resolves.toMatchObject({ - status: expect.any(String), - }); - await expect(getScanStatus("scan_missing", member.userId)).rejects.toThrow(/Scan not found/); + it("offers a scan folder only to the deployment owner and only when it exists", async () => { + const home = await tempDir(); + await files(home, ["file.txt"]); + await expect(onboardingScanFolder(member, home)).resolves.toBeNull(); + await expect(onboardingScanFolder(owner, undefined)).resolves.toBeNull(); + await expect(onboardingScanFolder(owner, path.join(home, "missing"))).resolves.toBeNull(); + await expect(onboardingScanFolder(owner, path.join(home, "file.txt"))).resolves.toBeNull(); + await expect(onboardingScanFolder(owner, home)).resolves.toBe(home); + await expect(startOnboardingScan(member, home)).rejects.toMatchObject({ code: "FORBIDDEN" }); }); - it("never reads the host without a configured scan home", async () => { - const { scanId } = await startOnboardingScan({ actor: owner }, {}); + it("keeps a scan with the account that started it and returns only relative paths", async () => { + const home = await tempDir(); + await project(home, "code/atlas", ["index.ts"]); + const { scanId } = await startOnboardingScan(owner, home); + const status = await completed(scanId, owner.userId); - expect(status.status).toBe("complete"); - expect(status.result).toMatchObject({ localRepos: [], fileSummary: { totalFiles: 0 } }); + expect(status).toEqual({ + status: "complete", + projects: [{ id: "project-0", name: "atlas", path: "code/atlas" }], + facts: [{ id: "language:TypeScript", label: "TypeScript" }], + }); + expect(JSON.stringify(status)).not.toContain(home); + expect(() => onboardingScanStatus(scanId, member.userId)).toThrow( + expect.objectContaining({ code: "NOT_FOUND" }), + ); + expect(() => onboardingScanStatus("missing-scan", owner.userId)).toThrow( + expect.objectContaining({ code: "NOT_FOUND" }), + ); + await expect( + applyOnboardingScan(untouchedPrisma, member, home, { + scanId, + projectIds: ["project-0"], + factIds: [], + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); }); - it("lets only the deployment owner discover the configured scan home", async () => { - // The shared test setup pins the emulated runtime, which also keeps the scan simulated. - vi.stubEnv("AGENT_RUNTIME", ""); - vi.stubEnv("SANDBOX_PROVIDER", ""); - const scanHome = await scanHomeWithOneFile(); + it("checks ownership, the folder and every chosen project again before writing", async () => { + const home = await tempDir(); + const outside = await tempDir(); + await project(home, "code/atlas", ["index.ts"]); + await project(home, "code/garden", ["main.py"]); + await project(outside, "elsewhere", ["main.rs"]); + const { scanId } = await startOnboardingScan(owner, home); + await completed(scanId, owner.userId); + const choice = { scanId, projectIds: ["project-1"], factIds: ["language:Python"] }; - const asMember = await startOnboardingScan({ actor: member, scanHome }, {}); - const memberStatus = await completed(asMember.scanId, member.userId); - expect(memberStatus.result).toMatchObject({ fileSummary: { totalFiles: 0 } }); + await expect( + applyOnboardingScan(untouchedPrisma, { ...owner, isDeploymentOwner: false }, home, choice), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + await expect( + applyOnboardingScan(untouchedPrisma, owner, outside, choice), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); - const asOwner = await startOnboardingScan({ actor: owner, scanHome }, {}); - const ownerStatus = await completed(asOwner.scanId, owner.userId); - expect(ownerStatus.status).toBe("complete"); - expect(ownerStatus.result).toMatchObject({ fileSummary: { totalFiles: 1 } }); + // The chosen project became a symlink out of the scan folder after the scan. + await rm(path.join(home, "code/garden"), { recursive: true }); + await symlink(path.join(outside, "elsewhere"), path.join(home, "code/garden")); + await expect(applyOnboardingScan(untouchedPrisma, owner, home, choice)).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: "That folder does not exist or is not readable.", + }); }); }); diff --git a/apps/api/src/onboarding-scan.ts b/apps/api/src/onboarding-scan.ts index 8d1c786..504cc84 100644 --- a/apps/api/src/onboarding-scan.ts +++ b/apps/api/src/onboarding-scan.ts @@ -1,815 +1,435 @@ -// Onboarding scan — real filesystem scan, Git analysis, and profile compilation. -// Runs synchronously on the host machine (no sandbox needed). -import { spawn } from "node:child_process"; -import { access, readdir, stat } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import type { Actor } from "@rakazo/contracts"; +// Onboarding discovery. With the deployment owner's consent, it lists the code projects (git +// checkouts) in the configured scan folder and the languages and tools they use. It reads +// directory listings only, never file contents, and never follows a symlink, so nothing outside +// the folder is reached. Results stay with the account that started the scan, and nothing is +// created until the owner picks projects and facts from them. +import { randomUUID } from "node:crypto"; +import type { Dirent } from "node:fs"; +import { readdir, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import { ORPCError } from "@orpc/server"; +import type { + Actor, + OnboardingScanApplyInput, + OnboardingScanApplyResult, + OnboardingScanStatus, + UserFactCategory, +} from "@rakazo/contracts"; +import { ONBOARDING_SCAN_MAX_FACTS, ONBOARDING_SCAN_MAX_PROJECTS } from "@rakazo/contracts"; +import type { PrismaClient } from "@rakazo/db"; +import { + createSpaceForMember, + InvalidProjectWorkspacePathError, + InvalidSpaceNameError, + InvalidUserFactError, + SpaceLimitError, + updateUserProfile, + wireProjectWorkspace, +} from "@rakazo/db"; -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- +/** Project roots are looked for at most this deep below the scan folder. */ +const PROJECT_DEPTH = 4; +/** Bounds the walk of a large home folder. */ +const MAX_DIRECTORIES = 5_000; +/** Bounds the language count inside one project. */ +const MAX_FILES_PER_PROJECT = 5_000; +const MAX_FILE_DEPTH = 8; +const MAX_LANGUAGES_PER_PROJECT = 3; +const SCAN_TTL_MS = 60 * 60_000; -interface ScanResult { - localRepos: RepoInfo[]; - projectFolders: ProjectFolder[]; - fileSummary: FileSummary; - profileSummary: ProfileSummary; - agentBriefing: string; - createdProjects: CreatedProject[]; - subAgentRoles: SubAgentRole[]; -} +const SKIPPED_DIRECTORIES = new Set([ + "node_modules", + "dist", + "build", + "out", + "target", + "vendor", + "venv", + "__pycache__", + "Library", +]); -interface RepoInfo { - path: string; +const LANGUAGES = new Map( + Object.entries({ + ts: "TypeScript", + tsx: "TypeScript", + mts: "TypeScript", + cts: "TypeScript", + js: "JavaScript", + jsx: "JavaScript", + mjs: "JavaScript", + cjs: "JavaScript", + py: "Python", + go: "Go", + rs: "Rust", + java: "Java", + kt: "Kotlin", + kts: "Kotlin", + swift: "Swift", + rb: "Ruby", + php: "PHP", + c: "C", + h: "C", + cc: "C++", + cpp: "C++", + cxx: "C++", + hpp: "C++", + cs: "C#", + dart: "Dart", + ex: "Elixir", + exs: "Elixir", + scala: "Scala", + lua: "Lua", + }), +); + +const COMPOSE_FILES = new Set([ + "docker-compose.yml", + "docker-compose.yaml", + "compose.yml", + "compose.yaml", +]); + +interface FoundProject { + id: string; name: string; - remote: string | null; - branch: string; - lastCommit: string | null; - languages: Record; - sizeBytes: number; -} - -interface ProjectFolder { + /** Relative to the scan folder. */ path: string; - name: string; - type: "git" | "workspace" | "documents" | "media" | "music" | "code"; - fileCount: number; - modifiedAt: string; -} - -interface FileSummary { - totalFiles: number; - totalSizeBytes: number; - byType: Record; - byExtension: Record; -} - -interface ProfileSummary { - interests: string[]; - skills: string[]; - workStyle: string; - communication: string; languages: string[]; - techStack: string[]; + tools: string[]; } -interface CreatedProject { +interface FoundFact { id: string; - name: string; - spaceId: string; - description: string; + label: string; + category: UserFactCategory; + /** Stored as data for the user's bots, so it stays a plain English statement. */ + value: string; } -interface ScanRecord { +export interface OnboardingScanFindings { + projects: FoundProject[]; + facts: FoundFact[]; +} + +interface ScanRecord extends OnboardingScanFindings { id: string; - botId: string; ownerUserId: string; - status: "scanning" | "connecting" | "building" | "briefing" | "complete" | "failed"; - progress: number; - message: string; - result: string; - startedAt: Date; - completedAt: Date | null; + /** Real path of the scanned folder. */ + home: string; + status: "scanning" | "complete" | "failed"; + finishedAt: number | null; + applying: boolean; } -// --------------------------------------------------------------------------- -// Registry (in-memory, 1h TTL) -// --------------------------------------------------------------------------- +/** Per API process; a scan only lives as long as the onboarding that started it. */ +const scans = new Map(); -const scanRegistry = new Map(); - -const emptyScanResult = (): ScanResult => ({ - localRepos: [], - projectFolders: [], - fileSummary: { totalFiles: 0, totalSizeBytes: 0, byType: {}, byExtension: {} }, - profileSummary: { - interests: [], - skills: [], - workStyle: "structured", - communication: "direct", - languages: [], - techStack: [], - }, - agentBriefing: "Profile ready", - createdProjects: [], - subAgentRoles: [], -}); - -async function runSimulatedScan(record: ScanRecord) { - const phases: Array<{ status: ScanRecord["status"]; progress: number; message: string }> = [ - { status: "scanning", progress: 25, message: "Scanning local files and repositories..." }, - { status: "connecting", progress: 40, message: "Connecting to your services..." }, - { status: "building", progress: 60, message: "Building your profile..." }, - { status: "briefing", progress: 80, message: "Briefing your agent team..." }, - ]; - for (const phase of phases) { - record.status = phase.status; - record.progress = phase.progress; - record.message = phase.message; - await new Promise((resolve) => setTimeout(resolve, 400)); - } - record.status = "complete"; - record.progress = 100; - record.message = "Scan complete — profile ready"; - record.result = JSON.stringify(emptyScanResult()); - record.completedAt = new Date(); +/** Code-point order, so results do not depend on the server's locale. */ +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; } -export interface OnboardingScanRequest { - actor: Pick; - botId?: string; - /** Home-style directory to discover. Unset keeps the scan off the host filesystem. */ - scanHome?: string; +/** Most frequent first, then by name. */ +function byCount([leftName, left]: [string, number], [rightName, right]: [string, number]): number { + return right - left || compareText(leftName, rightName); } -export async function startOnboardingScan( - { actor, botId, scanHome }: OnboardingScanRequest, - prisma: any, -): Promise<{ scanId: string }> { - const scanId = `scan_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - const record: ScanRecord = { - id: scanId, - botId: botId ?? "", - ownerUserId: actor.userId, - status: "scanning", - progress: 0, - message: "Scanning your computer...", - result: "[]", - startedAt: new Date(), - completedAt: null, - }; - scanRegistry.set(scanId, record); - - // Run async - void (async () => { - try { - // The host is the deployment owner's machine: only their onboarding may discover it, and - // only inside the directory the deployment explicitly configured. - if ( - !scanHome || - !actor.isDeploymentOwner || - process.env.AGENT_RUNTIME === "scripted" || - process.env.SANDBOX_PROVIDER === "fake" - ) { - await runSimulatedScan(record); - return; - } - const result = await executeFullScan(scanHome, botId ?? "", prisma); - record.status = "complete"; - record.progress = 100; - record.message = "Scan complete — profile ready"; - record.result = JSON.stringify(result); - record.completedAt = new Date(); - } catch (error) { - record.status = "failed"; - record.message = error instanceof Error ? error.message : "Scan failed"; - record.completedAt = new Date(); - } - })(); - - return { scanId }; -} - -export async function getScanStatus( - scanId: string, - userId: string, -): Promise<{ - status: ScanRecord["status"]; - progress: number; - message: string; - result?: unknown; -}> { - const record = scanRegistry.get(scanId); - if (!record || record.ownerUserId !== userId) throw new Error("Scan not found"); - const result = record.result ? JSON.parse(record.result) : undefined; - return { status: record.status, progress: record.progress, message: record.message, result }; -} - -// --------------------------------------------------------------------------- -// Full scan pipeline -// --------------------------------------------------------------------------- - -async function executeFullScan(homeDir: string, _botId: string, prisma: any): Promise { - // Phase 1: Local file scan - const [fileSummary, projectFolders] = await scanFiles(homeDir); - - // Phase 2: Git repository discovery - const localRepos = await discoverGitRepos(homeDir); - - // Phase 3: External service analysis (GitHub, etc.) - // Phase 4: Profile compilation from findings - const profileSummary = compileProfile(localRepos, fileSummary, projectFolders); - - // Phase 6: Project folder creation - const createdProjects = await createProjectFolders(localRepos, prisma); - - // Phase 7: Sub-agent role creation - const subAgentRoles = generateSubAgentRoles(profileSummary, localRepos); - - // Phase 5: Agent briefing (includes sub-agent roles) - const agentBriefing = generateBriefing(localRepos, profileSummary, fileSummary, subAgentRoles); - - return { - localRepos, - projectFolders, - fileSummary, - profileSummary, - agentBriefing, - createdProjects, - subAgentRoles, - }; -} - -// --------------------------------------------------------------------------- -// Phase 1: File scan -// --------------------------------------------------------------------------- - -async function scanFiles(homeDir: string): Promise<[FileSummary, ProjectFolder[]]> { - const summary: FileSummary = { totalFiles: 0, totalSizeBytes: 0, byType: {}, byExtension: {} }; - const folders: ProjectFolder[] = []; - - // Scan common directories - const scanDirs = [ - "Projects", - "Documents", - "Pictures", - "Music", - "Downloads", - "Desktop", - "src", - "code", - "work", - "data", - "repos", - "dev", - "workspace", - ]; - - for (const subDir of scanDirs) { - const dirPath = join(homeDir, subDir); - try { - await access(dirPath); - const stats = await scanDirectory(dirPath, homeDir, subDir); - summary.totalFiles += stats.totalFiles; - summary.totalSizeBytes += stats.totalSizeBytes; - Object.assign(summary.byType, stats.byType); - Object.assign(summary.byExtension, stats.byExtension); - - if (stats.totalFiles > 0) { - folders.push({ - path: dirPath, - name: subDir, - type: classifyFolderType(subDir), - fileCount: stats.totalFiles, - modifiedAt: new Date().toISOString(), - }); - } - } catch { - /* directory doesn't exist, skip */ - } - } - - return [summary, folders]; -} - -interface DirStats { - totalFiles: number; - totalSizeBytes: number; - byType: Record; - byExtension: Record; -} - -async function scanDirectory( - dirPath: string, - _homeDir: string, - _relativePath: string, -): Promise { - const stats: DirStats = { totalFiles: 0, totalSizeBytes: 0, byType: {}, byExtension: {} }; - +async function listDirectory(directory: string): Promise { try { - const entries = await readdir(dirPath, { withFileTypes: true }); + return (await readdir(directory, { withFileTypes: true })).sort((left, right) => + compareText(left.name, right.name), + ); + } catch { + return []; + } +} +/** isDirectory() is false for a symlink, so links (and anything they point to) are never walked. */ +function walkable(entry: Dirent): boolean { + return entry.isDirectory() && !entry.name.startsWith(".") && !SKIPPED_DIRECTORIES.has(entry.name); +} + +async function findProjectRoots(home: string): Promise { + const roots: string[] = []; + let visited = 0; + let level = [home]; + // Breadth first, so shallow projects win when a limit is reached. The scan folder itself is + // never a project: a home folder tracked by git still holds the projects below it. + for (let depth = 0; depth <= PROJECT_DEPTH && level.length > 0; depth += 1) { + const next: string[] = []; + for (const directory of level) { + if (roots.length >= ONBOARDING_SCAN_MAX_PROJECTS || visited >= MAX_DIRECTORIES) return roots; + visited += 1; + const entries = await listDirectory(directory); + if (depth > 0 && entries.some((entry) => entry.name === ".git")) { + roots.push(directory); + continue; + } + for (const entry of entries) { + if (walkable(entry)) next.push(path.join(directory, entry.name)); + } + } + level = next; + } + return roots; +} + +async function analyzeProject(root: string): Promise<{ languages: string[]; tools: string[] }> { + const counts = new Map(); + const tools = new Set(); + let files = 0; + const rootEntries = await listDirectory(root); + if (rootEntries.some((entry) => entry.name === ".github" && entry.isDirectory())) { + const github = await listDirectory(path.join(root, ".github")); + if (github.some((entry) => entry.name === "workflows" && entry.isDirectory())) { + tools.add("GitHub Actions"); + } + } + const visit = async (directory: string, depth: number, entries: Dirent[]): Promise => { for (const entry of entries) { - const fullPath = join(dirPath, entry.name); - - if (entry.isDirectory()) { - // Skip node_modules, .git, etc. - if (entry.name === "node_modules" || entry.name === ".git" || entry.name === ".venv") - continue; - const subStats = await scanDirectory(fullPath, _homeDir, _relativePath); - stats.totalFiles += subStats.totalFiles; - stats.totalSizeBytes += subStats.totalSizeBytes; - Object.assign(stats.byType, subStats.byType); - Object.assign(stats.byExtension, subStats.byExtension); - } else if (entry.isFile()) { - try { - const fileStat = await stat(fullPath); - stats.totalFiles++; - stats.totalSizeBytes += fileStat.size; - - // Classify by type - const type = classifyFileType(entry.name); - stats.byType[type] = (stats.byType[type] || 0) + 1; - - // Classify by extension - const ext = entry.name.split(".").pop()?.toLowerCase() || "none"; - stats.byExtension[ext] = (stats.byExtension[ext] || 0) + 1; - } catch { - /* permission denied */ + if (files >= MAX_FILES_PER_PROJECT) return; + if (entry.isFile()) { + files += 1; + const name = entry.name.toLowerCase(); + const extension = name.includes(".") ? name.slice(name.lastIndexOf(".") + 1) : ""; + const language = LANGUAGES.get(extension); + if (language) counts.set(language, (counts.get(language) ?? 0) + 1); + if (name === "dockerfile" || name.startsWith("dockerfile.") || COMPOSE_FILES.has(name)) { + tools.add("Docker"); } + if (extension === "tf") tools.add("Terraform"); + } else if (depth < MAX_FILE_DEPTH && walkable(entry)) { + const child = path.join(directory, entry.name); + await visit(child, depth + 1, await listDirectory(child)); } } - } catch { - /* permission denied */ - } - - return stats; + }; + await visit(root, 0, rootEntries); + const total = [...counts.values()].reduce((sum, count) => sum + count, 0); + const languages = [...counts.entries()] + .filter(([, count]) => count >= total * 0.1) + .sort(byCount) + .slice(0, MAX_LANGUAGES_PER_PROJECT) + .map(([language]) => language); + return { languages, tools: [...tools].sort() }; } -function classifyFileType(filename: string): string { - const ext = filename.split(".").pop()?.toLowerCase() || ""; - const codeExts = [ - "ts", - "tsx", - "js", - "jsx", - "py", - "go", - "rs", - "java", - "c", - "cpp", - "h", - "rb", - "php", - "swift", - "kt", - ]; - const docExts = ["md", "txt", "doc", "docx", "pdf", "rst", "tex"]; - const imageExts = ["png", "jpg", "jpeg", "gif", "svg", "ico", "webp", "bmp"]; - const musicExts = ["mp3", "wav", "flac", "aac", "ogg", "m4a", "wma"]; - - if (codeExts.includes(ext)) return "code"; - if (docExts.includes(ext)) return "documents"; - if (imageExts.includes(ext)) return "images"; - if (musicExts.includes(ext)) return "music"; - if (["json", "yaml", "yml", "toml", "cfg", "ini", "env", "sh", "bash", "zsh"].includes(ext)) - return "config"; - return "other"; +/** Labels used by the most projects first, then by name. */ +function rankLabels(lists: string[][]): string[] { + const counts = new Map(); + for (const list of lists) + for (const label of list) counts.set(label, (counts.get(label) ?? 0) + 1); + return [...counts.entries()].sort(byCount).map(([label]) => label); } -function classifyFolderType(name: string): ProjectFolder["type"] { - const lower = name.toLowerCase(); - if (lower.includes("music")) return "music"; - if (lower.includes("picture") || lower.includes("image") || lower.includes("photo")) - return "media"; - if (lower.includes("document") || lower.includes("doc")) return "documents"; - if ( - lower.includes("project") || - lower.includes("code") || - lower.includes("src") || - lower.includes("dev") || - lower.includes("workspace") || - lower.includes("repo") - ) - return "code"; - return "workspace"; +/** Lists the projects below a real (symlink-free) folder path, and what they use. */ +export async function discoverOnboardingScan(home: string): Promise { + const roots = await findProjectRoots(home); + const analyzed = await Promise.all( + roots.map(async (root) => ({ root, ...(await analyzeProject(root)) })), + ); + const projects = analyzed + .map(({ root, languages, tools }) => ({ + name: path.basename(root), + path: path.relative(home, root), + languages, + tools, + })) + .sort((left, right) => compareText(left.path, right.path)) + .map((project, index) => ({ id: `project-${index}`, ...project })); + const facts: FoundFact[] = [ + ...rankLabels(projects.map((project) => project.languages)).map((label) => ({ + id: `language:${label}`, + label, + category: "work" as const, + value: `Codes in ${label}`, + })), + ...rankLabels(projects.map((project) => project.tools)).map((label) => ({ + id: `tool:${label}`, + label, + category: "tools" as const, + value: `Uses ${label}`, + })), + ].slice(0, ONBOARDING_SCAN_MAX_FACTS); + return { projects, facts }; } -// --------------------------------------------------------------------------- -// Phase 2: Git repository discovery -// --------------------------------------------------------------------------- - -async function discoverGitRepos(homeDir: string): Promise { - const repos: RepoInfo[] = []; - +/** + * The scan reads the server's filesystem, so only the deployment owner may scan, and only the + * folder the deployment configured. Null hides the step. + */ +export async function onboardingScanFolder( + actor: Pick, + scanHome: string | undefined, +): Promise { + if (!actor.isDeploymentOwner || !scanHome) return null; try { - const gitDirs = await findGitDirectories(homeDir); - - for (const repoDir of gitDirs) { - try { - const repo = await analyzeGitRepo(repoDir, homeDir); - if (repo) repos.push(repo); - } catch { - /* skip broken repos */ - } - } - } catch { - /* git not available */ - } - - return repos; -} - -async function findGitDirectories(dir: string, depth: number = 0): Promise { - if (depth > 4) return []; // Limit search depth - const results: string[] = []; - - try { - const entries = await readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = join(dir, entry.name); - if (entry.isDirectory()) { - if (entry.name === ".git") { - results.push(dirname(fullPath)); - } else if ( - entry.name !== "node_modules" && - entry.name !== ".git" && - entry.name !== ".venv" && - entry.name !== "dist" - ) { - results.push(...(await findGitDirectories(fullPath, depth + 1))); - } - } - } - } catch { - /* permission denied */ - } - - return results; -} - -async function analyzeGitRepo(repoDir: string, _homeDir: string): Promise { - try { - const name = repoDir.split("/").pop() || ""; - - // Get remote URL - const remoteResult = await spawnPromise("git", ["remote", "get-url", "origin"], { - cwd: repoDir, - }).catch(() => ({ stdout: "" })); - const remote = remoteResult.stdout.trim() || null; - - // Get current branch - const branchResult = await spawnPromise("git", ["branch", "--show-current"], { - cwd: repoDir, - }).catch(() => ({ stdout: "" })); - const branch = branchResult.stdout.trim() || "unknown"; - - // Get last commit - const commitResult = await spawnPromise("git", ["log", "-1", "--format=%H"], { - cwd: repoDir, - }).catch(() => ({ stdout: "" })); - const lastCommit = commitResult.stdout.trim() || null; - - // Get language stats via git ls-files + wc - const languages = await getRepoLanguages(repoDir); - - // Get directory size - const sizeResult = await spawnPromise("du", ["-sb", repoDir]).catch(() => ({ stdout: "0" })); - const sizeBytes = parseInt(sizeResult.stdout.split("\t")[0] || "0", 10); - - return { path: repoDir, name, remote, branch, lastCommit, languages, sizeBytes }; + const home = await realpath(scanHome); + return (await stat(home)).isDirectory() ? home : null; } catch { return null; } } -async function getRepoLanguages(repoDir: string): Promise> { - const langs: Record = {}; - try { - const { stdout } = await spawnPromise("git", ["ls-files"], { cwd: repoDir }).catch(() => ({ - stdout: "", - })); - const files = stdout.trim().split("\n").filter(Boolean); - for (const file of files) { - const ext = file.split(".").pop()?.toLowerCase() || ""; - const langMap: Record = { - ts: "TypeScript", - tsx: "TypeScript", - js: "JavaScript", - jsx: "JavaScript", - py: "Python", - go: "Go", - rs: "Rust", - java: "Java", - c: "C", - cpp: "C++", - rb: "Ruby", - php: "PHP", - swift: "Swift", - kt: "Kotlin", - }; - const lang = langMap[ext] || ext || "unknown"; - langs[lang] = (langs[lang] || 0) + 1; - } - } catch { - /* no languages */ +function forgetExpiredScans(now: number): void { + for (const [id, scan] of scans) { + if (scan.finishedAt !== null && now - scan.finishedAt > SCAN_TTL_MS) scans.delete(id); } - return langs; } -// --------------------------------------------------------------------------- -// Phase 3: Profile compilation -// --------------------------------------------------------------------------- - -function compileProfile( - repos: RepoInfo[], - fileSummary: FileSummary, - _folders: ProjectFolder[], -): ProfileSummary { - const interests: string[] = []; - const skills: string[] = []; - const techStack: string[] = []; - const languageSet = new Set(); - - // Extract tech stack from repos - for (const repo of repos) { - for (const lang of Object.keys(repo.languages)) { - techStack.push(lang); - languageSet.add(lang); - } - } - - // Extract interests from file types - for (const [type, count] of Object.entries(fileSummary.byType)) { - if (count > 5) { - interests.push(`${type} (${count} files)`); - } - } - - // Derive skills from languages - const langToSkill: Record = { - TypeScript: "TypeScript/JavaScript", - Python: "Python", - Go: "Go", - Rust: "Rust", - Java: "Java", - C: "C/C++", - "C++": "C/C++", - Ruby: "Ruby", - Swift: "Swift/Kotlin", - PHP: "PHP", +export async function startOnboardingScan( + actor: Pick, + scanHome: string | undefined, +): Promise<{ scanId: string }> { + const home = await onboardingScanFolder(actor, scanHome); + if (!home) throw new ORPCError("FORBIDDEN"); + forgetExpiredScans(Date.now()); + const running = [...scans.values()].find( + (scan) => scan.ownerUserId === actor.userId && scan.status === "scanning", + ); + if (running) return { scanId: running.id }; + const scan: ScanRecord = { + id: randomUUID(), + ownerUserId: actor.userId, + home, + status: "scanning", + projects: [], + facts: [], + finishedAt: null, + applying: false, }; - for (const lang of techStack) { - const skill = langToSkill[lang]; - if (skill && !skills.includes(skill)) skills.push(skill); - } + scans.set(scan.id, scan); + void discoverOnboardingScan(home) + .then((findings) => { + scan.projects = findings.projects; + scan.facts = findings.facts; + scan.status = "complete"; + }) + .catch(() => { + scan.status = "failed"; + }) + .finally(() => { + scan.finishedAt = Date.now(); + }); + return { scanId: scan.id }; +} - // Deduplicate - const uniqueInterests = [...new Set(interests)].slice(0, 10); - const uniqueSkills = [...new Set(skills)].slice(0, 15); - const uniqueTech = [...new Set(techStack)].sort(); +/** Another account's scan reads as missing. */ +function ownedScan(scanId: string, userId: string): ScanRecord { + const scan = scans.get(scanId); + if (!scan || scan.ownerUserId !== userId) throw new ORPCError("NOT_FOUND"); + return scan; +} +export function onboardingScanStatus(scanId: string, userId: string): OnboardingScanStatus { + const scan = ownedScan(scanId, userId); + if (scan.status !== "complete") return { status: scan.status }; return { - interests: uniqueInterests, - skills: uniqueSkills, - workStyle: - uniqueTech.length > 3 - ? "multi-language" - : uniqueTech[0] - ? `${uniqueTech[0]}-focused` - : "general", - communication: "structured", - languages: Array.from(languageSet), - techStack: uniqueTech, + status: "complete", + projects: scan.projects.map(({ id, name, path }) => ({ id, name, path })), + facts: scan.facts.map(({ id, label }) => ({ id, label })), }; } -// --------------------------------------------------------------------------- -// Phase 4: Agent briefing -// --------------------------------------------------------------------------- +const MISSING_FOLDER = "That folder does not exist or is not readable."; -function generateBriefing( - repos: RepoInfo[], - profile: ProfileSummary, - fileSummary: FileSummary, - subAgentRoles: SubAgentRole[], -): string { - const lines: string[] = []; - lines.push(`# User Profile Briefing`); - lines.push(``); - lines.push(`## Technical Profile`); - lines.push(`- **Tech Stack:** ${profile.techStack.join(", ") || "N/A"}`); - lines.push(`- **Languages:** ${profile.languages.join(", ") || "N/A"}`); - lines.push(`- **Work Style:** ${profile.workStyle}`); - lines.push(``); - lines.push(`## Local Repositories (${repos.length} found)`); - for (const repo of repos) { - lines.push( - `- **${repo.name}** — ${repo.branch}, ${Object.values(repo.languages).reduce((a, b) => a + b, 0)} files, ${repo.remote || "no remote"}`, - ); +/** Resolves a found project again, so a folder swapped for a symlink cannot leave the scan folder. */ +async function confinedProjectFolder(home: string, relative: string): Promise { + let folder: string; + try { + folder = await realpath(path.join(home, relative)); + } catch { + throw new InvalidProjectWorkspacePathError(MISSING_FOLDER); } - lines.push(``); - lines.push(`## File System Summary`); - lines.push(`- **Total Files:** ${fileSummary.totalFiles.toLocaleString()}`); - lines.push(`- **Total Size:** ${(fileSummary.totalSizeBytes / (1024 * 1024)).toFixed(0)} MB`); - lines.push( - `- **Categories:** ${Object.entries(fileSummary.byType) - .map(([t, c]) => `${t}(${c})`) - .join(", ")}`, - ); - lines.push(``); - lines.push(`## User Interests`); - lines.push(`- ${profile.interests.slice(0, 5).join("\n- ")}`); - lines.push(``); - lines.push(`## User Skills`); - lines.push(`- ${profile.skills.slice(0, 5).join("\n- ")}`); - lines.push(``); - lines.push(`## Assigned Specialist Roles`); - for (const role of subAgentRoles) { - lines.push(`### ${role.name}`); - lines.push(`- **Role:** ${role.role}`); - lines.push(`- **Description:** ${role.description}`); - lines.push(`- **Instructions:** ${role.instructions}`); - lines.push(`- **Skills:** ${role.skills.join(", ")}`); - lines.push(""); + const inside = path.relative(home, folder); + if (!inside || inside === ".." || inside.startsWith(`..${path.sep}`) || path.isAbsolute(inside)) { + throw new InvalidProjectWorkspacePathError(MISSING_FOLDER); } - lines.push(`---`); - lines.push( - `This profile should be used to tailor all future interactions, project recommendations, and agent assignments.`, - ); - return lines.join("\n"); + return folder; } -// --------------------------------------------------------------------------- -// Phase 5: Project folder creation -// --------------------------------------------------------------------------- +async function projectSpace( + prisma: PrismaClient, + actor: Pick, + name: string, + folder: string, +): Promise<{ id: string; name: string }> { + // Applying twice, or onboarding again, reuses the user's own space linked to this folder. + const existing = await prisma.space.findFirst({ + where: { + workspacePath: folder, + deletingAt: null, + memberships: { some: { userId: actor.userId } }, + }, + select: { id: true, name: true }, + orderBy: { createdAt: "asc" }, + }); + if (existing) return existing; + const space = await createSpaceForMember(prisma, { + currentSpaceId: actor.spaceId, + userId: actor.userId, + name: name.slice(0, 60), + }); + await wireProjectWorkspace(prisma, { spaceId: space.id, workspacePath: folder }); + return space; +} -async function createProjectFolders(repos: RepoInfo[], prisma: any): Promise { - const created: CreatedProject[] = []; +function toRpcError(error: unknown): unknown { + if ( + error instanceof SpaceLimitError || + error instanceof InvalidSpaceNameError || + error instanceof InvalidProjectWorkspacePathError || + error instanceof InvalidUserFactError + ) { + return new ORPCError("BAD_REQUEST", { message: error.message }); + } + return error; +} - for (const repo of repos) { - try { - // Check if space already exists - const existing = await prisma.space.findFirst({ where: { title: repo.name } }); - if (existing) { - created.push({ - id: existing.id, - name: repo.name, - spaceId: existing.id, - description: repo.remote || repo.path, - }); - continue; - } - // Create new space - const space = await prisma.space.create({ - data: { title: repo.name, path: repo.path }, - }); - created.push({ - id: space.id, - name: repo.name, - spaceId: space.id, - description: repo.remote || repo.path, - }); - } catch { - // Space might already exist or DB unavailable — skip +/** Creates a project space per chosen project and saves the chosen facts to the profile. */ +export async function applyOnboardingScan( + prisma: PrismaClient, + actor: Pick, + scanHome: string | undefined, + input: OnboardingScanApplyInput, +): Promise { + const scan = ownedScan(input.scanId, actor.userId); + if (scan.status !== "complete" || scan.applying) throw new ORPCError("CONFLICT"); + scan.applying = true; + try { + // Ownership or the configured folder can change after the scan; check again before writing. + if ((await onboardingScanFolder(actor, scanHome)) !== scan.home) { + throw new ORPCError("FORBIDDEN"); } + const projectIds = new Set(input.projectIds); + const factIds = new Set(input.factIds); + const projects = scan.projects.filter((project) => projectIds.has(project.id)); + const facts = scan.facts.filter((fact) => factIds.has(fact.id)); + // Every chosen folder is checked before anything is written. + const chosen = await Promise.all( + projects.map(async (project) => ({ + name: project.name, + folder: await confinedProjectFolder(scan.home, project.path), + })), + ); + const added = + facts.length > 0 + ? await updateUserProfile(prisma, actor.userId, { + addFacts: facts.map(({ category, value }) => ({ + category, + value, + source: "onboarding" as const, + })), + }) + : []; + const spaces: Array<{ id: string; name: string }> = []; + for (const { name, folder } of chosen) + spaces.push(await projectSpace(prisma, actor, name, folder)); + return { spaces, factsAdded: added.filter((fact) => fact.created).length }; + } catch (error) { + throw toRpcError(error); + } finally { + scan.applying = false; } - - return created; } - -// --------------------------------------------------------------------------- -// Phase 6: Sub-agent role creation -// --------------------------------------------------------------------------- - -interface SubAgentRole { - name: string; - role: string; - description: string; - instructions: string; - skills: string[]; -} - -function generateSubAgentRoles(profile: ProfileSummary, repos: RepoInfo[]): SubAgentRole[] { - const roles: SubAgentRole[] = []; - const hasCode = profile.languages?.length > 0; - const hasDesign = profile.interests?.some( - (i) => i.toLowerCase().includes("design") || i.toLowerCase().includes("ui"), - ); - const hasData = profile.interests?.some( - (i) => - i.toLowerCase().includes("data") || - i.toLowerCase().includes("analytics") || - i.toLowerCase().includes("database"), - ); - const hasDevOps = profile.interests?.some( - (i) => - i.toLowerCase().includes("infra") || - i.toLowerCase().includes("ops") || - i.toLowerCase().includes("deploy"), - ); - const hasMusic = profile.interests?.some((i) => i.toLowerCase().includes("music")); - const hasWriting = profile.interests?.some( - (i) => i.toLowerCase().includes("write") || i.toLowerCase().includes("blog"), - ); - - if (hasCode) { - const primaryLang = profile.languages?.[0] || "TypeScript"; - roles.push({ - name: "Code Engineer", - role: "software-engineer", - description: `Specializes in ${primaryLang} development and code review`, - instructions: `You are the Code Engineer specialist. Your expertise is in ${profile.languages?.join(", ") || "software development"}. Review code, write tests, debug issues, and implement features. You have access to the user's local repositories and can run commands to build and test code.`, - skills: profile.skills.filter( - (s) => s.toLowerCase().includes("code") || s.toLowerCase().includes("program"), - ), - }); - } - - if (hasDesign) { - roles.push({ - name: "UI/UX Designer", - role: "ui-designer", - description: "Creates polished interfaces and design systems", - instructions: - "You are the UI/UX Designer specialist. Focus on creating beautiful, accessible interfaces. Review designs, suggest improvements, and ensure consistency across the user's projects.", - skills: ["design", "UI/UX"], - }); - } - - if (hasData) { - roles.push({ - name: "Data Analyst", - role: "data-analyst", - description: "Analyzes data, creates reports, and extracts insights", - instructions: - "You are the Data Analyst specialist. Process data from the user's projects, create visualizations, generate reports, and provide actionable insights.", - skills: ["data analysis", "statistics"], - }); - } - - if (hasDevOps) { - roles.push({ - name: "DevOps Engineer", - role: "devops-engineer", - description: "Manages infrastructure, deployments, and CI/CD", - instructions: - "You are the DevOps Engineer specialist. Handle deployments, manage infrastructure, configure CI/CD pipelines, and ensure system reliability.", - skills: ["DevOps", "infrastructure"], - }); - } - - if (hasMusic) { - roles.push({ - name: "Music Producer", - role: "music-producer", - description: "Assists with music production and audio engineering", - instructions: - "You are the Music Producer specialist. Help with music composition, audio engineering, mixing, and mastering. You understand the user's music projects and can provide expert advice.", - skills: ["music production", "audio engineering"], - }); - } - - if (hasWriting) { - roles.push({ - name: "Content Writer", - role: "content-writer", - description: "Creates and edits written content", - instructions: - "You are the Content Writer specialist. Write blog posts, documentation, articles, and other content for the user's projects and personal use.", - skills: ["writing", "content creation"], - }); - } - - // Always include a general assistant - roles.unshift({ - name: "General Assistant", - role: "general-assistant", - description: "Handles general tasks and coordinates other specialists", - instructions: `You are the General Assistant and team coordinator. Your role is to handle general queries, coordinate with other specialists (${ - roles - .slice(1) - .map((r) => r.name) - .join(", ") || "none yet" - }), and manage the user's daily workflow. You have access to all tools and can delegate tasks to specialists when needed.`, - skills: ["task management", "coordination"], - }); - - return roles; -} - -// --------------------------------------------------------------------------- -// Utility -// --------------------------------------------------------------------------- - -function spawnPromise( - cmd: string, - args: string[], - options?: { cwd?: string }, -): Promise<{ stdout: string; stderr: string }> { - return new Promise((resolve, reject) => { - const child = spawn(cmd, args, { ...options, timeout: 15000 }); - let stdout = ""; - let stderr = ""; - child.stdout?.on("data", (d: Buffer) => { - stdout += d.toString(); - }); - child.stderr?.on("data", (d: Buffer) => { - stderr += d.toString(); - }); - child.on("error", () => reject(new Error("spawn failed"))); - child.on("close", (code) => { - if (code === 0 || code === null) resolve({ stdout, stderr }); - else reject(new Error(`exit ${code}`)); - }); - }); -} - -// Keep old registry cleanup -setInterval(() => { - const cutoff = Date.now() - 3600000; - for (const [id, r] of scanRegistry) { - if (r.completedAt && r.completedAt.getTime() < cutoff) scanRegistry.delete(id); - } -}, 60000); diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 2baf21e..2af5c9a 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -157,6 +157,7 @@ import { resolveBusyBotName, toComputerStatus, } from "./computer-status.js"; +import { recordConnectedApp } from "./connected-app-facts.js"; import { listInbox } from "./inbox.js"; import { searchIntegrationCatalog } from "./integration-catalog.js"; import { @@ -180,7 +181,12 @@ import { promptFocus, startOnboarding, } from "./onboarding.js"; -import { getScanStatus, startOnboardingScan } from "./onboarding-scan.js"; +import { + applyOnboardingScan, + onboardingScanFolder, + onboardingScanStatus, + startOnboardingScan, +} from "./onboarding-scan.js"; import { listSpaceRuns } from "./runs.js"; import { addScreenProxyCapability } from "./screen-proxy.js"; import { querySpaceSearch } from "./search.js"; @@ -285,6 +291,11 @@ async function reconcilePendingConnections( : []), ]; if (updates.length > 0) await prisma.$transaction(updates); + const connected = new Set(sync.connectIds); + for (const row of rows) { + if (row.status !== "pending" || !connected.has(row.id)) continue; + await recordConnectedApp(prisma, { userId: owner.userId, connectorId, provider: row.provider }); + } } /** Serialize begin/revoke for one user+provider so slug-wide remote deletes cannot race a new connect. */ @@ -3434,16 +3445,18 @@ export function createRouter(deps: RouterDeps) { ); return { ok: true as const }; }), - scan: authed.onboarding.scan.handler(async ({ context, input }) => { - const { scanId } = await startOnboardingScan( - { actor: context.actor, botId: input.botId, scanHome: deps.env.onboardingScanHome }, - deps.prisma, - ); - return { scanId }; - }), - scanStatus: authed.onboarding.scanStatus.handler(async ({ context, input }) => { - return await getScanStatus(input.scanId, context.actor.userId); - }), + scanTarget: authed.onboarding.scanTarget.handler(async ({ context }) => ({ + folder: await onboardingScanFolder(context.actor, deps.env.onboardingScanHome), + })), + scan: authed.onboarding.scan.handler(({ context }) => + startOnboardingScan(context.actor, deps.env.onboardingScanHome), + ), + scanStatus: authed.onboarding.scanStatus.handler(async ({ context, input }) => + onboardingScanStatus(input.scanId, context.actor.userId), + ), + applyScan: authed.onboarding.applyScan.handler(({ context, input }) => + applyOnboardingScan(deps.prisma, context.actor, deps.env.onboardingScanHome, input), + ), }, integrationSetup: { get: authed.integrationSetup.get.handler(async ({ context }) => { @@ -3692,6 +3705,13 @@ export function createRouter(deps: RouterDeps) { } throw new IsolationError(); } + if (!auth.authorizationUrl) { + await recordConnectedApp(deps.prisma, { + userId: context.actor.userId, + connectorId: input.connectorId, + provider: input.provider, + }); + } return { connectionId: row.id, authorizationUrl: auth.authorizationUrl }; } catch (error) { if (error instanceof IsolationError) throw error; @@ -3997,6 +4017,13 @@ export function createRouter(deps: RouterDeps) { }, { timeout: 60_000 }, ); + if (row.status === "connected") { + await recordConnectedApp(deps.prisma, { + userId: context.actor.userId, + connectorId: row.connectorId, + provider: row.provider, + }); + } } return { id: row.id, diff --git a/apps/web/e2e/onboarding-discovery.spec.ts b/apps/web/e2e/onboarding-discovery.spec.ts new file mode 100644 index 0000000..0f639bc --- /dev/null +++ b/apps/web/e2e/onboarding-discovery.spec.ts @@ -0,0 +1,76 @@ +/** + * E2E: the deployment owner's onboarding asks before scanning the fixture home folder the + * harness configures, then adds only the projects and facts the user picks. + */ +import { expect, type Page, test } from "@playwright/test"; +import type { SpaceNavigation, UserProfileState } from "@rakazo/contracts"; +import { captureScreenshot, rpc, signup } from "./helpers"; + +const api = process.env.API_URL ?? "http://127.0.0.1:3110"; + +/** Advances whichever wizard step is showing until the scan step is reached. */ +async function walkToScanStep(page: Page) { + const surface = page.locator('[data-rakazo-surface="onboarding"]'); + await expect(async () => { + if ((await surface.getAttribute("data-onboarding-step")) === "scan") return; + const skip = page.getByRole("button", { name: /^Skip/ }).first(); + const next = (await skip.isVisible()) + ? skip + : page.getByRole("button", { name: "Continue", exact: true }); + await next.click({ timeout: 2_000 }); + await expect(surface).toHaveAttribute("data-onboarding-step", "scan", { timeout: 1_000 }); + }).toPass({ timeout: 60_000 }); +} + +test("asks before looking for projects and adds only what the user picks", async ({ + page, +}, testInfo) => { + const email = `discovery-${Date.now()}@rakazo.test`; + await signup(page, email, "password12", "Discovery"); + await page.waitForURL(/\/onboarding/, { timeout: 20_000 }); + // The harness starts without a deployment owner; this spec takes the seat for its run. + const owner = await page.request.post(`${api}/__e2e/deployment-owner`, { data: { email } }); + expect(owner.ok()).toBe(true); + const { previousOwnerUserId } = (await owner.json()) as { previousOwnerUserId: string | null }; + try { + await page.goto("/onboarding", { waitUntil: "domcontentloaded" }); + await walkToScanStep(page); + + await expect(page.getByRole("heading", { name: "Look for projects here?" })).toBeVisible(); + await expect(page.getByText(/rakazo-e2e-scan-home-/)).toBeVisible(); + await captureScreenshot(page, testInfo, "onboarding-discovery-consent"); + await page.getByRole("button", { name: "Look", exact: true }).click(); + + const projects = page.getByRole("group", { name: "Add as projects" }); + await expect(projects.getByRole("checkbox")).toHaveCount(2); + const atlas = projects.getByRole("checkbox", { name: /atlas/ }); + await expect(atlas).not.toBeChecked(); + await expect(projects.getByRole("checkbox", { name: /garden/ })).not.toBeChecked(); + await atlas.check(); + const facts = page.getByRole("group", { name: "Add to About you" }); + await expect(facts.getByRole("checkbox", { name: "TypeScript" })).toBeChecked(); + await facts.getByRole("checkbox", { name: "Python" }).uncheck(); + await captureScreenshot(page, testInfo, "onboarding-discovery-results"); + await page.getByRole("button", { name: "Add", exact: true }).click(); + await page.waitForURL(/\/app\/g\/[^/]+$/, { timeout: 30_000 }); + + const navigation = await rpc(page, "spaces/list", {}); + const names = navigation.spaces.map((space) => space.name); + expect(names).toContain("atlas"); + expect(names).not.toContain("garden"); + expect(navigation.spaces.find((space) => space.name === "atlas")?.workspace?.path).toMatch( + /code\/atlas$/, + ); + const values = (await rpc(page, "profile/get", {})).facts.map( + (fact) => fact.value, + ); + expect(values).toEqual( + expect.arrayContaining(["Codes in TypeScript", "Uses Docker", "Uses GitHub Actions"]), + ); + expect(values).not.toContain("Codes in Python"); + } finally { + await page.request.post(`${api}/__e2e/deployment-owner`, { + data: { userId: previousOwnerUserId }, + }); + } +}); diff --git a/apps/web/src/components/onboarding/OnboardingScanProgress.tsx b/apps/web/src/components/onboarding/OnboardingScanProgress.tsx deleted file mode 100644 index 16c41b4..0000000 --- a/apps/web/src/components/onboarding/OnboardingScanProgress.tsx +++ /dev/null @@ -1,186 +0,0 @@ -import { useEffect, useState } from "react"; -import { rpc } from "../../lib/rpc"; - -interface ScanStatus { - status: "scanning" | "connecting" | "building" | "briefing" | "complete" | "failed"; - progress: number; - message: string; - result?: unknown; -} - -const PHASES = [ - { - key: "scanning" as const, - label: "Scanning your computer...", - sublabel: "Files, folders, git repositories", - }, - { - key: "connecting" as const, - label: "Connecting to your services...", - sublabel: "GitHub, email, calendar", - }, - { - key: "building" as const, - label: "Building your profile...", - sublabel: "Interests, skills, preferences", - }, - { - key: "briefing" as const, - label: "Setting up your agent team...", - sublabel: "Project folders, role assignments", - }, -] as const; - -export function OnboardingScanProgress({ - onComplete, - botId, -}: { - onComplete: () => void; - botId?: string; -}) { - const [status, setStatus] = useState(null); - const [error, setError] = useState(null); - const [, setScanId] = useState(null); - - useEffect(() => { - let cancelled = false; - let pollingInterval: ReturnType | null = null; - - const run = async () => { - try { - const scanResult = await rpc.onboarding.scan(botId ? { botId } : {}); - if (cancelled) return; - setScanId(scanResult.scanId); - - // Poll for status - const poll = async () => { - try { - const s = await rpc.onboarding.scanStatus({ scanId: scanResult.scanId }); - if (cancelled) return; - setStatus(s); - if (s.status === "complete") { - onComplete(); - } else if (s.status === "failed") { - setError(s.message); - } - } catch { - // poll error — keep trying - } - }; - - // First poll immediately, then every 2s - await poll(); - pollingInterval = setInterval(poll, 2000); - } catch (err) { - if (!cancelled) setError(err instanceof Error ? err.message : "Scan failed"); - } - }; - - run(); - return () => { - cancelled = true; - if (pollingInterval) clearInterval(pollingInterval); - }; - }, [botId, onComplete]); - - if (error) { - return ( -
-

{error}

- -
- ); - } - - if (!status) { - return ( -
-
-

Preparing your profile...

-
- ); - } - - const currentPhaseIndex = PHASES.findIndex((p) => p.key === status.status); - const isComplete = status.status === "complete"; - - return ( -
-

- {isComplete ? "All set!" : "Getting to know you..."} -

-

- {isComplete - ? "Your profile and agent team are ready." - : "We're scanning your computer and services to personalize your experience."} -

- -
- {PHASES.map((phase, idx) => { - const isActive = idx === currentPhaseIndex && !isComplete; - const isDone = idx < currentPhaseIndex || isComplete; - return ( -
-
- {isDone ? ( - - ✓ - - ) : isActive ? ( - - ● - - ) : ( - - ○ - - )} -
-
-

- {phase.label} -

-

{phase.sublabel}

-
- {isActive && ( -
-
-
-
-
- )} -
- ); - })} -
- - {isComplete && ( -
- -
- )} -
- ); -} diff --git a/apps/web/src/components/onboarding/OnboardingScanStep.tsx b/apps/web/src/components/onboarding/OnboardingScanStep.tsx new file mode 100644 index 0000000..1b700c8 --- /dev/null +++ b/apps/web/src/components/onboarding/OnboardingScanStep.tsx @@ -0,0 +1,250 @@ +import { Trans, useLingui } from "@lingui/react/macro"; +import type { OnboardingScanFact, OnboardingScanProject } from "@rakazo/contracts"; +import { Button, Checkbox } from "@rakazo/ui-web"; +import { useEffect, useId, useRef, useState } from "react"; +import { rpc } from "../../lib/rpc"; +import { LoadingState } from "../ai/primitives"; + +const POLL_MS = 1_000; + +interface Found { + projects: OnboardingScanProject[]; + facts: OnboardingScanFact[]; +} + +function toggled(ids: string[], id: string, on: boolean): string[] { + return on ? [...ids, id] : ids.filter((entry) => entry !== id); +} + +/** + * Asks before looking at the server's scan folder, then lets the user choose which found projects + * become project spaces and which languages and tools go to About you. Nothing is created without + * a choice. Skips itself when this account cannot scan. + */ +export function OnboardingScanStep({ onDone }: { onDone: () => void }) { + const { t } = useLingui(); + const ids = useId(); + const done = useRef(onDone); + done.current = onDone; + const [folder, setFolder] = useState(null); + const [looking, setLooking] = useState(false); + const [scanId, setScanId] = useState(null); + const [found, setFound] = useState(null); + const [failed, setFailed] = useState(false); + const [projectIds, setProjectIds] = useState([]); + const [factIds, setFactIds] = useState([]); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + rpc.onboarding + .scanTarget() + .then((target) => { + if (cancelled) return; + if (target.folder) setFolder(target.folder); + else done.current(); + }) + .catch(() => { + if (!cancelled) done.current(); + }); + return () => { + cancelled = true; + }; + }, []); + + useEffect(() => { + if (!scanId) return; + let cancelled = false; + let timer: ReturnType | undefined; + const poll = () => { + rpc.onboarding + .scanStatus({ scanId }) + .then((status) => { + if (cancelled) return; + if (status.status === "scanning") { + timer = setTimeout(poll, POLL_MS); + } else if (status.status === "failed") { + setFailed(true); + } else { + setFound({ projects: status.projects, facts: status.facts }); + setFactIds(status.facts.map((fact) => fact.id)); + } + }) + .catch(() => { + if (!cancelled) setFailed(true); + }); + }; + poll(); + return () => { + cancelled = true; + clearTimeout(timer); + }; + }, [scanId]); + + async function look() { + setLooking(true); + try { + setScanId((await rpc.onboarding.scan()).scanId); + } catch { + setFailed(true); + } + } + + async function add() { + if (!scanId) return; + setSaving(true); + setError(null); + try { + await rpc.onboarding.applyScan({ scanId, projectIds, factIds }); + done.current(); + } catch (reason) { + setError(reason instanceof Error ? reason.message : String(reason)); + setSaving(false); + } + } + + const continueButton = ( +
+ +
+ ); + + if (failed) { + return ( +
+

+ Could not look in this folder. +

+ {continueButton} +
+ ); + } + + if (found && found.projects.length === 0) { + return ( +
+

+ No projects found +

+ {continueButton} +
+ ); + } + + if (found) { + return ( +
+
+ +

+ Add as projects +

+ +
+ {found.projects.map((project) => ( + + ))} +
+
+ + {found.facts.length > 0 ? ( +
+ + Add to About you + +
+ {found.facts.map((fact) => ( + + ))} +
+
+ ) : null} + + {error ?

{error}

: null} + +
+ + +
+
+ ); + } + + if (looking) { + return ( +
+ +
+ ); + } + + if (!folder) return null; + + return ( +
+

+ Look for projects here? +

+

+ {folder} +

+

+ Only file and folder names are read. +

+
+ + +
+
+ ); +} diff --git a/apps/web/src/pages/Onboarding.tsx b/apps/web/src/pages/Onboarding.tsx index 0423eaa..75b9533 100644 --- a/apps/web/src/pages/Onboarding.tsx +++ b/apps/web/src/pages/Onboarding.tsx @@ -9,7 +9,7 @@ import { type OnboardingEmojiConfig, OnboardingOoDesigner, } from "../components/onboarding/OnboardingOoDesigner"; -import { OnboardingScanProgress } from "../components/onboarding/OnboardingScanProgress"; +import { OnboardingScanStep } from "../components/onboarding/OnboardingScanStep"; import { KOKORO_LOCAL_TOKEN, OnboardingVoiceStep, @@ -115,7 +115,6 @@ export function OnboardingPage() { const [profilePersisted, setProfilePersisted] = useState(false); const [crewReady, setCrewReady] = useState(false); const [_helpVibe, setHelpVibe] = useState(null); - const [scanBotId, setScanBotId] = useState(null); const [emojiConfig, setEmojiConfig] = useState(defaultEmojiConfig); const [prefs, setPrefs] = useState(defaultPreferences()); @@ -141,14 +140,6 @@ export function OnboardingPage() { if (cancelled) return; setBootstrapMe(me); setHelpVibe(inferHelpVibeHints(collectLocalProfileSignals(me))); - // Get bot ID for scan phase - try { - const botList = await rpc.bots.list(); - const mainBot = botList.find((b) => b.kind === "main"); - if (mainBot && !cancelled) setScanBotId(mainBot.id); - } catch { - /* best effort */ - } setIntegrationSetup(integrations); // The first step renders before this resolves; never pull a user back who moved on. if (!me.needsModel && integrations?.needsSetup) { @@ -513,12 +504,7 @@ export function OnboardingPage() {
) : null} - {step === "scan" ? ( - setStep("bot")} - botId={scanBotId ?? undefined} - /> - ) : null} + {step === "scan" ? setStep("bot")} /> : null} {step === "bot" && error ? (
diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 3865e1b..af399a3 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -10,6 +10,7 @@ export * from "./integration-settings.js"; export * from "./local-settings.js"; export * from "./mcp.js"; export * from "./model-tiers.js"; +export * from "./onboarding-scan.js"; export * from "./openai-compatible-ui.js"; export * from "./project-goals.js"; export * from "./project-sessions.js"; diff --git a/packages/contracts/src/onboarding-scan.ts b/packages/contracts/src/onboarding-scan.ts new file mode 100644 index 0000000..97faa35 --- /dev/null +++ b/packages/contracts/src/onboarding-scan.ts @@ -0,0 +1,49 @@ +import * as z from "zod"; +import { Id } from "./ids.js"; + +/** Upper bounds on one scan's choices; the server lists at most this many projects and facts. */ +export const ONBOARDING_SCAN_MAX_PROJECTS = 30; +export const ONBOARDING_SCAN_MAX_FACTS = 20; + +/** The folder a consented scan would read, or null when this account cannot scan. */ +export const OnboardingScanTargetSchema = z.object({ folder: z.string().nullable() }); +export type OnboardingScanTarget = z.infer; + +export const OnboardingScanProjectSchema = z.object({ + id: z.string(), + name: z.string(), + /** Relative to the scanned folder. */ + path: z.string(), +}); +export type OnboardingScanProject = z.infer; + +export const OnboardingScanFactSchema = z.object({ + id: z.string(), + /** A language or tool name, e.g. "TypeScript". */ + label: z.string(), +}); +export type OnboardingScanFact = z.infer; + +export const OnboardingScanStatusSchema = z.discriminatedUnion("status", [ + z.object({ status: z.literal("scanning") }), + z.object({ status: z.literal("failed") }), + z.object({ + status: z.literal("complete"), + projects: z.array(OnboardingScanProjectSchema), + facts: z.array(OnboardingScanFactSchema), + }), +]); +export type OnboardingScanStatus = z.infer; + +export const OnboardingScanApplyInputSchema = z.object({ + scanId: z.string(), + projectIds: z.array(z.string()).max(ONBOARDING_SCAN_MAX_PROJECTS), + factIds: z.array(z.string()).max(ONBOARDING_SCAN_MAX_FACTS), +}); +export type OnboardingScanApplyInput = z.infer; + +export const OnboardingScanApplyResultSchema = z.object({ + spaces: z.array(z.object({ id: Id, name: z.string() })), + factsAdded: z.number().int().nonnegative(), +}); +export type OnboardingScanApplyResult = z.infer; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 6640d4c..48506ba 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -83,6 +83,12 @@ import { IntegrationSetupStateSchema, } from "./integration-settings.js"; import { ModelTierMappingSchema, SetModelTierInput } from "./model-tiers.js"; +import { + OnboardingScanApplyInputSchema, + OnboardingScanApplyResultSchema, + OnboardingScanStatusSchema, + OnboardingScanTargetSchema, +} from "./onboarding-scan.js"; import { CreateProjectGoalInput, ProjectGoalSchema, @@ -676,15 +682,13 @@ export const appContract = { z.object({ botId: Id, provider: z.string(), connectorId: z.string().default("composio") }), ) .output(z.object({ ok: z.literal(true) })), - scan: oc.input(z.object({ botId: Id.optional() })).output(z.object({ scanId: z.string() })), - scanStatus: oc.input(z.object({ scanId: z.string() })).output( - z.object({ - status: z.enum(["scanning", "connecting", "building", "briefing", "complete", "failed"]), - progress: z.number(), - message: z.string(), - result: z.any().optional(), - }), - ), + /** The folder a scan would read; null when this account cannot scan, so the step is skipped. */ + scanTarget: oc.output(OnboardingScanTargetSchema), + /** Start scanning that folder. Only called after the user agreed. */ + scan: oc.output(z.object({ scanId: z.string() })), + scanStatus: oc.input(z.object({ scanId: z.string() })).output(OnboardingScanStatusSchema), + /** Create a project space per chosen project and save the chosen facts. */ + applyScan: oc.input(OnboardingScanApplyInputSchema).output(OnboardingScanApplyResultSchema), }, integrationSetup: { get: oc.output(IntegrationSetupStateSchema), diff --git a/packages/testkit/src/authorization.test.ts b/packages/testkit/src/authorization.test.ts index ce1e785..21ffadd 100644 --- a/packages/testkit/src/authorization.test.ts +++ b/packages/testkit/src/authorization.test.ts @@ -556,19 +556,18 @@ describeWithDatabase("API authorization and resource isolation", () => { ).toMatchObject({ content: "owner-only-memory" }); // The user profile is per user: another user's fact ids and spaces are out of reach. - const ownerProfile = await rpc<{ facts: Array<{ id: string; status: string }> }>( - app, - owner, - "profile/update", - { - addFacts: [ - { category: "work", value: "Owner works nights" }, - { category: "health", value: "Owner health note", source: "onboarding" }, - ], - }, - ); - const ownerFactIds = ownerProfile.facts.map((fact) => fact.id); - expect(ownerProfile.facts.map((fact) => fact.status).sort()).toEqual(["active", "pending"]); + const ownerProfile = await rpc<{ + facts: Array<{ id: string; status: string; source: string }>; + }>(app, owner, "profile/update", { + addFacts: [ + { category: "work", value: "Owner works nights" }, + { category: "health", value: "Owner health note", source: "onboarding" }, + ], + }); + // Connecting an app above already remembered it; this check is about the facts added here. + const ownerFacts = ownerProfile.facts.filter((fact) => fact.source !== "connector"); + const ownerFactIds = ownerFacts.map((fact) => fact.id); + expect(ownerFacts.map((fact) => fact.status).sort()).toEqual(["active", "pending"]); const profileCalls = [ ["profile/update", { confirmFactIds: ownerFactIds }], ["profile/update", { editFacts: [{ id: ownerFactIds[0], value: "Stolen" }] }], diff --git a/packages/testkit/src/cli/harness.ts b/packages/testkit/src/cli/harness.ts index 27415a4..5305a76 100644 --- a/packages/testkit/src/cli/harness.ts +++ b/packages/testkit/src/cli/harness.ts @@ -1,9 +1,11 @@ import { execSync } from "node:child_process"; -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import path from "node:path"; import { loadRootEnv } from "@rakazo/core/node/load-root-env"; import { PostgreSqlContainer } from "@testcontainers/postgresql"; import type { createApp } from "../../../../apps/api/src/app.ts"; +import { writeOnboardingScanFixture } from "../onboarding-scan-fixture.js"; import { runProcess } from "./process.js"; loadRootEnv(); @@ -106,6 +108,7 @@ async function main() { "packages/testkit/src/bot-secrets.test.ts", "packages/testkit/src/delegation.postgres.test.ts", "packages/testkit/src/user-profile.postgres.test.ts", + "packages/testkit/src/onboarding-discovery.postgres.test.ts", "packages/db/src/space-membership.postgres.test.ts", "packages/db/src/main-agent.postgres.test.ts", "packages/db/src/user-profile.postgres.test.ts", @@ -180,6 +183,10 @@ async function main() { { fetch: thirdParties.fetch, resolveHostname: thirdParties.resolveHostname }, ); const email = new EmailEmulator(); + // A fixture home folder, so the deployment owner's onboarding offers the real project scan. + const scanHome = await mkdtemp(path.join(tmpdir(), "rakazo-e2e-scan-home-")); + await writeOnboardingScanFixture(scanHome); + process.env.ONBOARDING_SCAN_HOME = scanHome; const handles = await createApp({ databaseUrl, prisma: undefined, @@ -192,13 +199,30 @@ async function main() { }, integrationsCatalogUrl: "https://catalog.example.test/", }); + // No spec user owns the deployment unless it asks (POST /__e2e/deployment-owner), so + // owner-only onboarding steps appear only in the specs that test them, whatever runs first. + await handles.prisma.deploymentSettings.upsert({ + where: { id: "default" }, + create: { + id: "default", + ownerUserId: E2E_NO_OWNER, + signupsEnabled: true, + signupAllowlist: "", + signupPolicyInitialized: true, + }, + update: { ownerUserId: E2E_NO_OWNER }, + }); let activeRequests = 0; const requestWaiters = new Set<() => void>(); const server = serve({ fetch: async (request) => { - if (new URL(request.url).pathname === "/__e2e/emails") { + const { pathname } = new URL(request.url); + if (pathname === "/__e2e/emails") { return Response.json(email.sent, { headers: { "cache-control": "no-store" } }); } + if (pathname === "/__e2e/deployment-owner" && request.method === "POST") { + return Response.json(await setDeploymentOwner(handles, await request.json())); + } activeRequests += 1; try { return await handles.app.fetch(request); @@ -266,6 +290,7 @@ async function main() { await new Promise((resolve) => requestWaiters.add(resolve)); } await handles.stop().catch(() => undefined); + await rm(scanHome, { recursive: true, force: true }).catch(() => undefined); for (let index = 0; index < computers.length; index += 4) { const results = await Promise.allSettled( computers.slice(index, index + 4).map((computer) => @@ -304,6 +329,9 @@ async function main() { type AppHandles = Awaited>; +/** Matches no user, so the first signup does not claim the deployment. */ +const E2E_NO_OWNER = "e2e-no-owner"; + async function managedComputers(handles: AppHandles) { if (!["e2b", "daytona", "box"].includes(sandboxProvider)) return []; return handles.prisma.computer.findMany({ @@ -312,6 +340,33 @@ async function managedComputers(handles: AppHandles) { }); } +/** + * Lets a spec act as the deployment owner (by email) for owner-only steps, then restore the + * previous owner (by id). Returns the owner it replaced. + */ +async function setDeploymentOwner( + handles: AppHandles, + input: { email?: string; userId?: string | null }, +): Promise<{ previousOwnerUserId: string | null }> { + const previous = await handles.prisma.deploymentSettings.findUnique({ + where: { id: "default" }, + select: { ownerUserId: true }, + }); + const ownerUserId = input.email + ? ( + await handles.prisma.user.findUniqueOrThrow({ + where: { email: input.email }, + select: { id: true }, + }) + ).id + : (input.userId ?? E2E_NO_OWNER); + await handles.prisma.deploymentSettings.update({ + where: { id: "default" }, + data: { ownerUserId }, + }); + return { previousOwnerUserId: previous?.ownerUserId ?? null }; +} + async function writeSummary(reportDir: string, summary: Record) { await writeFile( path.join(reportDir, "summary.json"), diff --git a/packages/testkit/src/onboarding-discovery.postgres.test.ts b/packages/testkit/src/onboarding-discovery.postgres.test.ts new file mode 100644 index 0000000..671131b --- /dev/null +++ b/packages/testkit/src/onboarding-discovery.postgres.test.ts @@ -0,0 +1,363 @@ +import { mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs"; +import { mkdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + ComposioEmulator, + PipedreamConnector, + ThirdPartyConnectorEmulator, +} from "@rakazo/adapters"; +import type { + OnboardingScanApplyResult, + OnboardingScanStatus, + SpaceNavigation, + UserProfileState, +} from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import type { createApp } from "../../../apps/api/src/app.ts"; +import { sessionCookieHeader } from "./index.js"; +import { writeOnboardingScanFixture } from "./onboarding-scan-fixture.js"; + +type App = { request: (input: string, init?: RequestInit) => Response | Promise }; +type AppHandles = Awaited>; +type Actor = { userId: string; spaceId: string }; + +process.env.WAKEUP_DRIVER = "memory"; +process.env.SANDBOX_PROVIDER = "fake"; +process.env.AGENT_RUNTIME = "scripted"; + +const hasDb = process.env.VERIFY_DATABASE === "1" && Boolean(process.env.DATABASE_URL); +const describeWithDatabase = hasDb ? describe : describe.skip; +const TEST_ENCRYPTION_KEY = "offline-discovery-test-encryption-key"; + +describeWithDatabase("onboarding discovery through the API", () => { + let handles: AppHandles; + let app: App; + let composio: ComposioEmulator; + const stamp = `${Date.now()}-${Math.random().toString(36).slice(2)}`; + const dataDir = mkdtempSync(path.join(tmpdir(), "rakazo-discovery-")); + const home = realpathSync(mkdtempSync(path.join(tmpdir(), "rakazo-scan-home-"))); + const outside = realpathSync(mkdtempSync(path.join(tmpdir(), "rakazo-scan-outside-"))); + + beforeAll(async () => { + await writeOnboardingScanFixture(home); + const { createApp } = await import("../../../apps/api/src/app.ts"); + composio = new ComposioEmulator(); + const thirdParties = new ThirdPartyConnectorEmulator(); + handles = await createApp({ + databaseUrl: process.env.DATABASE_URL!, + dataDir, + sandboxProvider: "fake", + agentRuntime: "scripted", + wakeupDriver: "memory", + signupsEnabled: "true", + encryptionKey: TEST_ENCRYPTION_KEY, + onboardingScanHome: home, + composio, + pipedream: new PipedreamConnector( + { + clientId: "fake-client-id", + clientSecret: "fake-client-secret", + projectId: "fake-project-id", + environment: "development", + identitySecret: TEST_ENCRYPTION_KEY, + }, + { fetch: thirdParties.fetch, resolveHostname: thirdParties.resolveHostname }, + ), + remoteConnectors: { + fetch: thirdParties.fetch, + resolveHostname: thirdParties.resolveHostname, + }, + }); + app = handles.app; + }); + + afterAll(async () => { + await handles?.stop(); + for (const dir of [dataDir, home, outside]) rmSync(dir, { recursive: true, force: true }); + }); + + async function account(label: string): Promise<{ cookie: string; actor: Actor }> { + const cookie = await signup(app, `${label}-${stamp}@rakazo.test`, label); + return { cookie, actor: await rpc(app, cookie, "me") }; + } + + async function makeDeploymentOwner(actor: Actor): Promise { + await handles.prisma.deploymentSettings.update({ + where: { id: "default" }, + data: { ownerUserId: actor.userId }, + }); + } + + async function finishedScan(cookie: string) { + const { scanId } = await rpc<{ scanId: string }>(app, cookie, "onboarding/scan"); + const status = await vi.waitFor( + async () => { + const current = await rpc(app, cookie, "onboarding/scanStatus", { + scanId, + }); + if (current.status === "scanning") throw new Error("still scanning"); + return current; + }, + { timeout: 10_000, interval: 50 }, + ); + if (status.status !== "complete") throw new Error("scan failed"); + return { scanId, ...status }; + } + + it("offers the scan only to the deployment owner and keeps it with them", async () => { + const owner = await account("discovery-owner"); + const other = await account("discovery-other"); + await makeDeploymentOwner(owner.actor); + + await expect(rpc(app, other.cookie, "onboarding/scanTarget")).resolves.toEqual({ + folder: null, + }); + expect((await raw(app, other.cookie, "onboarding/scan")).status).toBe(403); + await expect(rpc(app, owner.cookie, "onboarding/scanTarget")).resolves.toEqual({ + folder: home, + }); + + const scan = await finishedScan(owner.cookie); + expect(scan.projects).toEqual([ + { id: "project-0", name: "atlas", path: "code/atlas" }, + { id: "project-1", name: "garden", path: "code/garden" }, + ]); + expect(scan.facts.map((fact) => fact.label)).toEqual([ + "Python", + "TypeScript", + "Docker", + "GitHub Actions", + ]); + expect( + (await raw(app, other.cookie, "onboarding/scanStatus", { scanId: scan.scanId })).status, + ).toBe(404); + expect( + ( + await raw(app, other.cookie, "onboarding/applyScan", { + scanId: scan.scanId, + projectIds: ["project-0"], + factIds: ["language:Python"], + }) + ).status, + ).toBe(404); + await expect(rpc(app, other.cookie, "profile/get")).resolves.toMatchObject({ + facts: [], + }); + }); + + it("creates the chosen projects in the owner's own spaces and saves only the chosen facts", async () => { + const owner = await account("discovery-creator"); + const intruder = await account("discovery-intruder"); + await makeDeploymentOwner(owner.actor); + const atlasFolder = path.join(home, "code/atlas"); + // Another tenant already linked a space with the same name to the same folder. + const foreign = await rpc<{ id: string }>(app, intruder.cookie, "spaces/create", { + name: "atlas", + workspacePath: atlasFolder, + }); + + const scan = await finishedScan(owner.cookie); + const choice = { + scanId: scan.scanId, + projectIds: ["project-0"], + factIds: ["language:TypeScript", "tool:Docker"], + }; + const applied = await rpc( + app, + owner.cookie, + "onboarding/applyScan", + choice, + ); + + expect(applied.factsAdded).toBe(2); + expect(applied.spaces).toEqual([{ id: expect.any(String), name: "atlas" }]); + const created = applied.spaces[0]!.id; + expect(created).not.toBe(foreign.id); + const ownerOrg = await handles.prisma.space.findUniqueOrThrow({ + where: { id: owner.actor.spaceId }, + select: { organizationId: true }, + }); + await expect( + handles.prisma.space.findUniqueOrThrow({ + where: { id: created }, + select: { organizationId: true, workspacePath: true }, + }), + ).resolves.toEqual({ organizationId: ownerOrg.organizationId, workspacePath: atlasFolder }); + await expect( + handles.prisma.spaceMember.findMany({ + where: { spaceId: { in: [created, foreign.id] } }, + select: { spaceId: true, userId: true, role: true }, + orderBy: { spaceId: "asc" }, + }), + ).resolves.toEqual( + expect.arrayContaining([ + { spaceId: created, userId: owner.actor.userId, role: "owner" }, + { spaceId: foreign.id, userId: intruder.actor.userId, role: "owner" }, + ]), + ); + expect( + await handles.prisma.spaceMember.count({ where: { spaceId: { in: [created, foreign.id] } } }), + ).toBe(2); + await expect( + handles.prisma.memoryDocument.count({ + where: { spaceId: created, userId: owner.actor.userId, scope: "user" }, + }), + ).resolves.toBe(1); + + const ownerSpaces = await rpc(app, owner.cookie, "spaces/list"); + expect(ownerSpaces.spaces).toContainEqual( + expect.objectContaining({ + id: created, + name: "atlas", + workspace: expect.objectContaining({ path: atlasFolder }), + }), + ); + expect(ownerSpaces.spaces.map((space) => space.id)).not.toContain(foreign.id); + const intruderSpaces = await rpc(app, intruder.cookie, "spaces/list"); + expect(intruderSpaces.spaces.map((space) => space.id)).not.toContain(created); + + const profile = await rpc(app, owner.cookie, "profile/get"); + expect( + profile.facts.map(({ category, value, source }) => ({ category, value, source })), + ).toEqual( + expect.arrayContaining([ + { category: "work", value: "Codes in TypeScript", source: "onboarding" }, + { category: "tools", value: "Uses Docker", source: "onboarding" }, + ]), + ); + expect(profile.facts).toHaveLength(2); + await expect(rpc(app, intruder.cookie, "profile/get")).resolves.toMatchObject( + { facts: [] }, + ); + + // Applying again reuses the space and the facts. + await expect( + rpc(app, owner.cookie, "onboarding/applyScan", choice), + ).resolves.toEqual({ spaces: [{ id: created, name: "atlas" }], factsAdded: 0 }); + expect(await handles.prisma.spaceMember.count({ where: { userId: owner.actor.userId } })).toBe( + 2, + ); + }); + + it("refuses a chosen project that left the scan folder and writes nothing", async () => { + const owner = await account("discovery-escape"); + await makeDeploymentOwner(owner.actor); + const scan = await finishedScan(owner.cookie); + await mkdir(path.join(outside, "elsewhere", ".git"), { recursive: true }); + await rm(path.join(home, "code/garden"), { recursive: true, force: true }); + symlinkSync(path.join(outside, "elsewhere"), path.join(home, "code/garden")); + + const response = await raw(app, owner.cookie, "onboarding/applyScan", { + scanId: scan.scanId, + projectIds: ["project-1"], + factIds: ["language:Python"], + }); + expect(response.status).toBe(400); + expect(await handles.prisma.spaceMember.count({ where: { userId: owner.actor.userId } })).toBe( + 1, + ); + await expect(rpc(app, owner.cookie, "profile/get")).resolves.toMatchObject({ + facts: [], + }); + }); + + it("remembers each connected app once as a tools fact and keeps it after revoking", async () => { + const user = await account("discovery-apps"); + const other = await account("discovery-apps-other"); + // A featured app is named by its label, whatever the account is called. + const gmail = await rpc<{ connectionId: string }>(app, user.cookie, "connections/begin", { + connectorId: "composio", + provider: "GMAIL", + displayName: "Personal", + }); + // Other apps take the first account's name; a second account adds nothing. + for (const displayName of ["GitHub", "GitHub 2"]) { + await rpc(app, user.cookie, "connections/begin", { + connectorId: "composio", + provider: "GITHUB", + displayName, + }); + } + // Connected through the authorization round trip. + const linear = await rpc<{ connectionId: string; authorizationUrl: string | null }>( + app, + user.cookie, + "connections/begin", + { connectorId: "pipedream", provider: "linear", displayName: "Linear" }, + ); + expect(linear.authorizationUrl).not.toBeNull(); + await expect( + rpc<{ status: string }>(app, user.cookie, "connections/complete", { + connectionId: linear.connectionId, + }), + ).resolves.toMatchObject({ status: "connected" }); + // Connected remotely, then noticed when the catalog reconciles a pending row. + await handles.prisma.connection.create({ + data: { + spaceId: user.actor.spaceId, + userId: user.actor.userId, + provider: "SLACK", + displayName: "SLACK", + status: "pending", + }, + }); + await composio.begin( + { provider: "SLACK", redirectUrl: "http://127.0.0.1.invalid/callback" }, + { + operationId: "discovery-test", + traceId: "discovery-test", + spaceId: user.actor.spaceId, + userId: user.actor.userId, + signal: new AbortController().signal, + }, + ); + await rpc(app, user.cookie, "connections/catalog", { connectorId: "composio" }); + await rpc(app, user.cookie, "connections/revoke", { connectionId: gmail.connectionId }); + + const profile = await rpc(app, user.cookie, "profile/get"); + expect( + profile.facts + .map(({ category, value, source }) => ({ category, value, source })) + .sort((left, right) => left.value.localeCompare(right.value)), + ).toEqual([ + { category: "tools", value: "Uses GitHub", source: "connector" }, + { category: "tools", value: "Uses Gmail", source: "connector" }, + { category: "tools", value: "Uses Linear", source: "connector" }, + { category: "tools", value: "Uses Slack", source: "connector" }, + ]); + await expect(rpc(app, other.cookie, "profile/get")).resolves.toMatchObject({ + facts: [], + }); + }); +}); + +async function signup(app: App, email: string, name: string) { + const response = await app.request("/api/auth/sign-up/email", { + method: "POST", + headers: { "content-type": "application/json", origin: "http://127.0.0.1:5173" }, + body: JSON.stringify({ email, password: "password12", name }), + }); + if (response.status >= 400) { + throw new Error(`signup failed ${response.status}: ${await response.text()}`); + } + return sessionCookieHeader(response); +} + +async function raw(app: App, cookie: string, procedure: string, body: unknown = {}) { + return app.request(`/rpc/${procedure}`, { + method: "POST", + headers: { "content-type": "application/json", cookie, origin: "http://127.0.0.1:5173" }, + body: JSON.stringify({ json: body }), + }); +} + +async function rpc(app: App, cookie: string, procedure: string, body: unknown = {}) { + const response = await raw(app, cookie, procedure, body); + const text = await response.text(); + const payload = JSON.parse(text) as { json?: T; error?: { message?: string } }; + if (response.status >= 400 || payload.error) { + throw new Error(`${procedure} ${response.status}: ${payload.error?.message ?? text}`); + } + return payload.json as T; +} diff --git a/packages/testkit/src/onboarding-scan-fixture.ts b/packages/testkit/src/onboarding-scan-fixture.ts new file mode 100644 index 0000000..f79e884 --- /dev/null +++ b/packages/testkit/src/onboarding-scan-fixture.ts @@ -0,0 +1,35 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +/** An empty but valid git directory, so git commands stop here instead of searching upwards. */ +async function writeGitDirectory(project: string): Promise { + await mkdir(path.join(project, ".git", "objects"), { recursive: true }); + await mkdir(path.join(project, ".git", "refs"), { recursive: true }); + await writeFile(path.join(project, ".git", "HEAD"), "ref: refs/heads/main\n"); +} + +async function writeFiles(root: string, files: Record): Promise { + for (const [relative, content] of Object.entries(files)) { + const file = path.join(root, relative); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, content); + } +} + +/** + * A home folder for the onboarding scan: two projects ("code/atlas" in TypeScript with Docker and + * GitHub Actions, "code/garden" in Python) and a notes folder that is not a project. + */ +export async function writeOnboardingScanFixture(home: string): Promise { + await writeFiles(home, { + "code/atlas/src/index.ts": "export const atlas = 1;\n", + "code/atlas/src/app.tsx": "export const App = () => null;\n", + "code/atlas/Dockerfile": "FROM scratch\n", + "code/atlas/.github/workflows/ci.yml": "on: push\n", + "code/garden/main.py": "print('garden')\n", + "code/garden/util.py": "VALUE = 1\n", + "notes/todo.md": "- water the plants\n", + }); + await writeGitDirectory(path.join(home, "code/atlas")); + await writeGitDirectory(path.join(home, "code/garden")); +}