// Onboarding discovery. With the deployment owner's consent, it lists the code projects (git // checkouts) in the configured scan folder and the languages and tools they use. It reads // directory listings only, never file contents, and never follows a symlink, so nothing outside // the folder is reached. Results stay with the account that started the scan, and nothing is // created until the owner picks projects and facts from them. import { randomUUID } from "node:crypto"; import type { Dirent } from "node:fs"; import { readdir, realpath, stat } from "node:fs/promises"; import path from "node:path"; import { ORPCError } from "@orpc/server"; import type { Actor, OnboardingScanApplyInput, OnboardingScanApplyResult, OnboardingScanStatus, UserFactCategory, } from "@rakazo/contracts"; import { ONBOARDING_SCAN_MAX_FACTS, ONBOARDING_SCAN_MAX_PROJECTS } from "@rakazo/contracts"; import type { PrismaClient } from "@rakazo/db"; import { createSpaceForMember, ensureMainConversation, InvalidProjectWorkspacePathError, InvalidSpaceNameError, InvalidUserFactError, resolveProjectWorkspaceDirectory, SpaceLimitError, updateUserProfile, wireProjectWorkspace, } from "@rakazo/db"; /** Project roots are looked for at most this deep below the scan folder. */ const PROJECT_DEPTH = 4; /** Bounds the walk of a large home folder. */ const MAX_DIRECTORIES = 5_000; /** Bounds the language count inside one project. */ const MAX_FILES_PER_PROJECT = 5_000; const MAX_FILE_DEPTH = 8; const MAX_LANGUAGES_PER_PROJECT = 3; const SCAN_TTL_MS = 60 * 60_000; const SKIPPED_DIRECTORIES = new Set([ "node_modules", "dist", "build", "out", "target", "vendor", "venv", "__pycache__", "Library", ]); const LANGUAGES = new Map( Object.entries({ ts: "TypeScript", tsx: "TypeScript", mts: "TypeScript", cts: "TypeScript", js: "JavaScript", jsx: "JavaScript", mjs: "JavaScript", cjs: "JavaScript", py: "Python", go: "Go", rs: "Rust", java: "Java", kt: "Kotlin", kts: "Kotlin", swift: "Swift", rb: "Ruby", php: "PHP", c: "C", h: "C", cc: "C++", cpp: "C++", cxx: "C++", hpp: "C++", cs: "C#", dart: "Dart", ex: "Elixir", exs: "Elixir", scala: "Scala", lua: "Lua", }), ); const COMPOSE_FILES = new Set([ "docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml", ]); interface FoundProject { id: string; name: string; /** Relative to the scan folder. */ path: string; languages: string[]; tools: string[]; } interface FoundFact { id: string; label: string; category: UserFactCategory; /** Stored as data for the user's bots, so it stays a plain English statement. */ value: string; } export interface OnboardingScanFindings { projects: FoundProject[]; facts: FoundFact[]; } interface ScanRecord extends OnboardingScanFindings { id: string; ownerUserId: string; /** Real path of the scanned folder. */ home: string; status: "scanning" | "complete" | "failed"; finishedAt: number | null; applying: boolean; } /** Per API process; a scan only lives as long as the onboarding that started it. */ const scans = new Map(); /** Code-point order, so results do not depend on the server's locale. */ function compareText(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; } /** Most frequent first, then by name. */ function byCount([leftName, left]: [string, number], [rightName, right]: [string, number]): number { return right - left || compareText(leftName, rightName); } async function listDirectory(directory: string): Promise { try { return (await readdir(directory, { withFileTypes: true })).sort((left, right) => compareText(left.name, right.name), ); } catch { return []; } } /** isDirectory() is false for a symlink, so links (and anything they point to) are never walked. */ function walkable(entry: Dirent): boolean { return entry.isDirectory() && !entry.name.startsWith(".") && !SKIPPED_DIRECTORIES.has(entry.name); } async function findProjectRoots(home: string): Promise { const roots: string[] = []; let visited = 0; let level = [home]; // Breadth first, so shallow projects win when a limit is reached. The scan folder itself is // never a project: a home folder tracked by git still holds the projects below it. for (let depth = 0; depth <= PROJECT_DEPTH && level.length > 0; depth += 1) { const next: string[] = []; for (const directory of level) { if (roots.length >= ONBOARDING_SCAN_MAX_PROJECTS || visited >= MAX_DIRECTORIES) return roots; visited += 1; const entries = await listDirectory(directory); if (depth > 0 && entries.some((entry) => entry.name === ".git")) { roots.push(directory); continue; } for (const entry of entries) { if (walkable(entry)) next.push(path.join(directory, entry.name)); } } level = next; } return roots; } async function analyzeProject(root: string): Promise<{ languages: string[]; tools: string[] }> { const counts = new Map(); const tools = new Set(); let files = 0; const rootEntries = await listDirectory(root); if (rootEntries.some((entry) => entry.name === ".github" && entry.isDirectory())) { const github = await listDirectory(path.join(root, ".github")); if (github.some((entry) => entry.name === "workflows" && entry.isDirectory())) { tools.add("GitHub Actions"); } } const visit = async (directory: string, depth: number, entries: Dirent[]): Promise => { for (const entry of entries) { if (files >= MAX_FILES_PER_PROJECT) return; if (entry.isFile()) { files += 1; const name = entry.name.toLowerCase(); const extension = name.includes(".") ? name.slice(name.lastIndexOf(".") + 1) : ""; const language = LANGUAGES.get(extension); if (language) counts.set(language, (counts.get(language) ?? 0) + 1); if (name === "dockerfile" || name.startsWith("dockerfile.") || COMPOSE_FILES.has(name)) { tools.add("Docker"); } if (extension === "tf") tools.add("Terraform"); } else if (depth < MAX_FILE_DEPTH && walkable(entry)) { const child = path.join(directory, entry.name); await visit(child, depth + 1, await listDirectory(child)); } } }; await visit(root, 0, rootEntries); const total = [...counts.values()].reduce((sum, count) => sum + count, 0); const languages = [...counts.entries()] .filter(([, count]) => count >= total * 0.1) .sort(byCount) .slice(0, MAX_LANGUAGES_PER_PROJECT) .map(([language]) => language); return { languages, tools: [...tools].sort() }; } /** Labels used by the most projects first, then by name. */ function rankLabels(lists: string[][]): string[] { const counts = new Map(); for (const list of lists) for (const label of list) counts.set(label, (counts.get(label) ?? 0) + 1); return [...counts.entries()].sort(byCount).map(([label]) => label); } /** Lists the projects below a real (symlink-free) folder path, and what they use. */ export async function discoverOnboardingScan(home: string): Promise { const roots = await findProjectRoots(home); const analyzed = await Promise.all( roots.map(async (root) => ({ root, ...(await analyzeProject(root)) })), ); const projects = analyzed .map(({ root, languages, tools }) => ({ name: path.basename(root), path: path.relative(home, root), languages, tools, })) .sort((left, right) => compareText(left.path, right.path)) .map((project, index) => ({ id: `project-${index}`, ...project })); const facts: FoundFact[] = [ ...rankLabels(projects.map((project) => project.languages)).map((label) => ({ id: `language:${label}`, label, category: "work" as const, value: `Codes in ${label}`, })), ...rankLabels(projects.map((project) => project.tools)).map((label) => ({ id: `tool:${label}`, label, category: "tools" as const, value: `Uses ${label}`, })), ].slice(0, ONBOARDING_SCAN_MAX_FACTS); return { projects, facts }; } /** * The scan reads the server's filesystem, so only the deployment owner may scan, and only the * folder the deployment configured. Null hides the step. */ export async function onboardingScanFolder( actor: Pick, scanHome: string | undefined, ): Promise { if (!actor.isDeploymentOwner || !scanHome) return null; try { const home = await realpath(scanHome); return (await stat(home)).isDirectory() ? home : null; } catch { return null; } } function forgetExpiredScans(now: number): void { for (const [id, scan] of scans) { if (scan.finishedAt !== null && now - scan.finishedAt > SCAN_TTL_MS) scans.delete(id); } } export async function startOnboardingScan( actor: Pick, scanHome: string | undefined, ): Promise<{ scanId: string }> { const home = await onboardingScanFolder(actor, scanHome); if (!home) throw new ORPCError("FORBIDDEN"); forgetExpiredScans(Date.now()); const running = [...scans.values()].find( (scan) => scan.ownerUserId === actor.userId && scan.status === "scanning", ); if (running) return { scanId: running.id }; const scan: ScanRecord = { id: randomUUID(), ownerUserId: actor.userId, home, status: "scanning", projects: [], facts: [], finishedAt: null, applying: false, }; scans.set(scan.id, scan); void discoverOnboardingScan(home) .then((findings) => { scan.projects = findings.projects; scan.facts = findings.facts; scan.status = "complete"; }) .catch(() => { scan.status = "failed"; }) .finally(() => { scan.finishedAt = Date.now(); }); return { scanId: scan.id }; } /** Another account's scan reads as missing. */ function ownedScan(scanId: string, userId: string): ScanRecord { const scan = scans.get(scanId); if (!scan || scan.ownerUserId !== userId) throw new ORPCError("NOT_FOUND"); return scan; } export function onboardingScanStatus(scanId: string, userId: string): OnboardingScanStatus { const scan = ownedScan(scanId, userId); if (scan.status !== "complete") return { status: scan.status }; return { status: "complete", projects: scan.projects.map(({ id, name, path }) => ({ id, name, path })), facts: scan.facts.map(({ id, label }) => ({ id, label })), }; } const MISSING_FOLDER = "That folder does not exist or is not readable."; /** Resolves a found project again, so a folder swapped for a symlink cannot leave the scan folder. */ async function confinedProjectFolder(home: string, relative: string): Promise { const folder = await resolveProjectWorkspaceDirectory(path.join(home, relative)); const inside = path.relative(home, folder); if (!inside || inside === ".." || inside.startsWith(`..${path.sep}`) || path.isAbsolute(inside)) { throw new InvalidProjectWorkspacePathError(MISSING_FOLDER); } return folder; } async function projectSpace( prisma: PrismaClient, actor: Pick, name: string, folder: string, ): Promise<{ id: string; name: string }> { // Applying twice, or onboarding again, reuses the user's own space linked to this folder. const existing = await prisma.space.findFirst({ where: { workspacePath: folder, deletingAt: null, memberships: { some: { userId: actor.userId } }, }, select: { id: true, name: true }, orderBy: { createdAt: "asc" }, }); if (existing) return existing; const space = await createSpaceForMember(prisma, { currentSpaceId: actor.spaceId, userId: actor.userId, name: name.slice(0, 60), }); await wireProjectWorkspace(prisma, { spaceId: space.id, workspacePath: folder }); // Like any new project, it starts with its own main agent. await ensureMainConversation(prisma, actor.userId, space.id); return space; } function toRpcError(error: unknown): unknown { if ( error instanceof SpaceLimitError || error instanceof InvalidSpaceNameError || error instanceof InvalidProjectWorkspacePathError || error instanceof InvalidUserFactError ) { return new ORPCError("BAD_REQUEST", { message: error.message }); } return error; } /** Creates a project space per chosen project and saves the chosen facts to the profile. */ export async function applyOnboardingScan( prisma: PrismaClient, actor: Pick, scanHome: string | undefined, input: OnboardingScanApplyInput, ): Promise { const scan = ownedScan(input.scanId, actor.userId); if (scan.status !== "complete" || scan.applying) throw new ORPCError("CONFLICT"); scan.applying = true; try { // Ownership or the configured folder can change after the scan; check again before writing. if ((await onboardingScanFolder(actor, scanHome)) !== scan.home) { throw new ORPCError("FORBIDDEN"); } const projectIds = new Set(input.projectIds); const factIds = new Set(input.factIds); const projects = scan.projects.filter((project) => projectIds.has(project.id)); const facts = scan.facts.filter((fact) => factIds.has(fact.id)); // Every chosen folder is checked before anything is written. const chosen = await Promise.all( projects.map(async (project) => ({ name: project.name, folder: await confinedProjectFolder(scan.home, project.path), })), ); const added = facts.length > 0 ? await updateUserProfile(prisma, actor.userId, { addFacts: facts.map(({ category, value }) => ({ category, value, source: "onboarding" as const, })), }) : []; const spaces: Array<{ id: string; name: string }> = []; for (const { name, folder } of chosen) spaces.push(await projectSpace(prisma, actor, name, folder)); return { spaces, factsAdded: added.filter((fact) => fact.created).length }; } catch (error) { throw toRpcError(error); } finally { scan.applying = false; } }