evOo-local/apps/api/src/onboarding-scan.ts

// Onboarding discovery. With the deployment owner's consent, it lists the code projects (git
// checkouts) in the configured scan folder and the languages and tools they use. It reads
// directory listings only, never file contents, and never follows a symlink, so nothing outside
// the folder is reached. Results stay with the account that started the scan, and nothing is
// created until the owner picks projects and facts from them.
import { randomUUID } from "node:crypto";
import type { Dirent } from "node:fs";
import { readdir, realpath, stat } from "node:fs/promises";
import path from "node:path";
import { ORPCError } from "@orpc/server";
import type {
  Actor,
  OnboardingScanApplyInput,
  OnboardingScanApplyResult,
  OnboardingScanStatus,
  UserFactCategory,
} from "@rakazo/contracts";
import { ONBOARDING_SCAN_MAX_FACTS, ONBOARDING_SCAN_MAX_PROJECTS } from "@rakazo/contracts";
import type { PrismaClient } from "@rakazo/db";
import {
  createSpaceForMember,
  ensureMainConversation,
  InvalidProjectWorkspacePathError,
  InvalidSpaceNameError,
  InvalidUserFactError,
  resolveProjectWorkspaceDirectory,
  SpaceLimitError,
  updateUserProfile,
  wireProjectWorkspace,
} from "@rakazo/db";

/** Project roots are looked for at most this deep below the scan folder. */
const PROJECT_DEPTH = 4;
/** Bounds the walk of a large home folder. */
const MAX_DIRECTORIES = 5_000;
/** Bounds the language count inside one project. */
const MAX_FILES_PER_PROJECT = 5_000;
const MAX_FILE_DEPTH = 8;
const MAX_LANGUAGES_PER_PROJECT = 3;
const SCAN_TTL_MS = 60 * 60_000;

const SKIPPED_DIRECTORIES = new Set([
  "node_modules",
  "dist",
  "build",
  "out",
  "target",
  "vendor",
  "venv",
  "__pycache__",
  "Library",
]);

const LANGUAGES = new Map<string, string>(
  Object.entries({
    ts: "TypeScript",
    tsx: "TypeScript",
    mts: "TypeScript",
    cts: "TypeScript",
    js: "JavaScript",
    jsx: "JavaScript",
    mjs: "JavaScript",
    cjs: "JavaScript",
    py: "Python",
    go: "Go",
    rs: "Rust",
    java: "Java",
    kt: "Kotlin",
    kts: "Kotlin",
    swift: "Swift",
    rb: "Ruby",
    php: "PHP",
    c: "C",
    h: "C",
    cc: "C++",
    cpp: "C++",
    cxx: "C++",
    hpp: "C++",
    cs: "C#",
    dart: "Dart",
    ex: "Elixir",
    exs: "Elixir",
    scala: "Scala",
    lua: "Lua",
  }),
);

const COMPOSE_FILES = new Set([
  "docker-compose.yml",
  "docker-compose.yaml",
  "compose.yml",
  "compose.yaml",
]);

interface FoundProject {
  id: string;
  name: string;
  /** Relative to the scan folder. */
  path: string;
  languages: string[];
  tools: string[];
}

interface FoundFact {
  id: string;
  label: string;
  category: UserFactCategory;
  /** Stored as data for the user's bots, so it stays a plain English statement. */
  value: string;
}

export interface OnboardingScanFindings {
  projects: FoundProject[];
  facts: FoundFact[];
}

interface ScanRecord extends OnboardingScanFindings {
  id: string;
  ownerUserId: string;
  /** Real path of the scanned folder. */
  home: string;
  status: "scanning" | "complete" | "failed";
  finishedAt: number | null;
  applying: boolean;
}

/** Per API process; a scan only lives as long as the onboarding that started it. */
const scans = new Map<string, ScanRecord>();

/** Code-point order, so results do not depend on the server's locale. */
function compareText(left: string, right: string): number {
  return left < right ? -1 : left > right ? 1 : 0;
}

/** Most frequent first, then by name. */
function byCount([leftName, left]: [string, number], [rightName, right]: [string, number]): number {
  return right - left || compareText(leftName, rightName);
}

async function listDirectory(directory: string): Promise<Dirent[]> {
  try {
    return (await readdir(directory, { withFileTypes: true })).sort((left, right) =>
      compareText(left.name, right.name),
    );
  } catch {
    return [];
  }
}

/** isDirectory() is false for a symlink, so links (and anything they point to) are never walked. */
function walkable(entry: Dirent): boolean {
  return entry.isDirectory() && !entry.name.startsWith(".") && !SKIPPED_DIRECTORIES.has(entry.name);
}

async function findProjectRoots(home: string): Promise<string[]> {
  const roots: string[] = [];
  let visited = 0;
  let level = [home];
  // Breadth first, so shallow projects win when a limit is reached. The scan folder itself is
  // never a project: a home folder tracked by git still holds the projects below it.
  for (let depth = 0; depth <= PROJECT_DEPTH && level.length > 0; depth += 1) {
    const next: string[] = [];
    for (const directory of level) {
      if (roots.length >= ONBOARDING_SCAN_MAX_PROJECTS || visited >= MAX_DIRECTORIES) return roots;
      visited += 1;
      const entries = await listDirectory(directory);
      if (depth > 0 && entries.some((entry) => entry.name === ".git")) {
        roots.push(directory);
        continue;
      }
      for (const entry of entries) {
        if (walkable(entry)) next.push(path.join(directory, entry.name));
      }
    }
    level = next;
  }
  return roots;
}

async function analyzeProject(root: string): Promise<{ languages: string[]; tools: string[] }> {
  const counts = new Map<string, number>();
  const tools = new Set<string>();
  let files = 0;
  const rootEntries = await listDirectory(root);
  if (rootEntries.some((entry) => entry.name === ".github" && entry.isDirectory())) {
    const github = await listDirectory(path.join(root, ".github"));
    if (github.some((entry) => entry.name === "workflows" && entry.isDirectory())) {
      tools.add("GitHub Actions");
    }
  }
  const visit = async (directory: string, depth: number, entries: Dirent[]): Promise<void> => {
    for (const entry of entries) {
      if (files >= MAX_FILES_PER_PROJECT) return;
      if (entry.isFile()) {
        files += 1;
        const name = entry.name.toLowerCase();
        const extension = name.includes(".") ? name.slice(name.lastIndexOf(".") + 1) : "";
        const language = LANGUAGES.get(extension);
        if (language) counts.set(language, (counts.get(language) ?? 0) + 1);
        if (name === "dockerfile" || name.startsWith("dockerfile.") || COMPOSE_FILES.has(name)) {
          tools.add("Docker");
        }
        if (extension === "tf") tools.add("Terraform");
      } else if (depth < MAX_FILE_DEPTH && walkable(entry)) {
        const child = path.join(directory, entry.name);
        await visit(child, depth + 1, await listDirectory(child));
      }
    }
  };
  await visit(root, 0, rootEntries);
  const total = [...counts.values()].reduce((sum, count) => sum + count, 0);
  const languages = [...counts.entries()]
    .filter(([, count]) => count >= total * 0.1)
    .sort(byCount)
    .slice(0, MAX_LANGUAGES_PER_PROJECT)
    .map(([language]) => language);
  return { languages, tools: [...tools].sort() };
}

/** Labels used by the most projects first, then by name. */
function rankLabels(lists: string[][]): string[] {
  const counts = new Map<string, number>();
  for (const list of lists)
    for (const label of list) counts.set(label, (counts.get(label) ?? 0) + 1);
  return [...counts.entries()].sort(byCount).map(([label]) => label);
}

/** Lists the projects below a real (symlink-free) folder path, and what they use. */
export async function discoverOnboardingScan(home: string): Promise<OnboardingScanFindings> {
  const roots = await findProjectRoots(home);
  const analyzed = await Promise.all(
    roots.map(async (root) => ({ root, ...(await analyzeProject(root)) })),
  );
  const projects = analyzed
    .map(({ root, languages, tools }) => ({
      name: path.basename(root),
      path: path.relative(home, root),
      languages,
      tools,
    }))
    .sort((left, right) => compareText(left.path, right.path))
    .map((project, index) => ({ id: `project-${index}`, ...project }));
  const facts: FoundFact[] = [
    ...rankLabels(projects.map((project) => project.languages)).map((label) => ({
      id: `language:${label}`,
      label,
      category: "work" as const,
      value: `Codes in ${label}`,
    })),
    ...rankLabels(projects.map((project) => project.tools)).map((label) => ({
      id: `tool:${label}`,
      label,
      category: "tools" as const,
      value: `Uses ${label}`,
    })),
  ].slice(0, ONBOARDING_SCAN_MAX_FACTS);
  return { projects, facts };
}

/**
 * The scan reads the server's filesystem, so only the deployment owner may scan, and only the
 * folder the deployment configured. Null hides the step.
 */
export async function onboardingScanFolder(
  actor: Pick<Actor, "isDeploymentOwner">,
  scanHome: string | undefined,
): Promise<string | null> {
  if (!actor.isDeploymentOwner || !scanHome) return null;
  try {
    const home = await realpath(scanHome);
    return (await stat(home)).isDirectory() ? home : null;
  } catch {
    return null;
  }
}

function forgetExpiredScans(now: number): void {
  for (const [id, scan] of scans) {
    if (scan.finishedAt !== null && now - scan.finishedAt > SCAN_TTL_MS) scans.delete(id);
  }
}

export async function startOnboardingScan(
  actor: Pick<Actor, "userId" | "isDeploymentOwner">,
  scanHome: string | undefined,
): Promise<{ scanId: string }> {
  const home = await onboardingScanFolder(actor, scanHome);
  if (!home) throw new ORPCError("FORBIDDEN");
  forgetExpiredScans(Date.now());
  const running = [...scans.values()].find(
    (scan) => scan.ownerUserId === actor.userId && scan.status === "scanning",
  );
  if (running) return { scanId: running.id };
  const scan: ScanRecord = {
    id: randomUUID(),
    ownerUserId: actor.userId,
    home,
    status: "scanning",
    projects: [],
    facts: [],
    finishedAt: null,
    applying: false,
  };
  scans.set(scan.id, scan);
  void discoverOnboardingScan(home)
    .then((findings) => {
      scan.projects = findings.projects;
      scan.facts = findings.facts;
      scan.status = "complete";
    })
    .catch(() => {
      scan.status = "failed";
    })
    .finally(() => {
      scan.finishedAt = Date.now();
    });
  return { scanId: scan.id };
}

/** Another account's scan reads as missing. */
function ownedScan(scanId: string, userId: string): ScanRecord {
  const scan = scans.get(scanId);
  if (!scan || scan.ownerUserId !== userId) throw new ORPCError("NOT_FOUND");
  return scan;
}

export function onboardingScanStatus(scanId: string, userId: string): OnboardingScanStatus {
  const scan = ownedScan(scanId, userId);
  if (scan.status !== "complete") return { status: scan.status };
  return {
    status: "complete",
    projects: scan.projects.map(({ id, name, path }) => ({ id, name, path })),
    facts: scan.facts.map(({ id, label }) => ({ id, label })),
  };
}

const MISSING_FOLDER = "That folder does not exist or is not readable.";

/** Resolves a found project again, so a folder swapped for a symlink cannot leave the scan folder. */
async function confinedProjectFolder(home: string, relative: string): Promise<string> {
  const folder = await resolveProjectWorkspaceDirectory(path.join(home, relative));
  const inside = path.relative(home, folder);
  if (!inside || inside === ".." || inside.startsWith(`..${path.sep}`) || path.isAbsolute(inside)) {
    throw new InvalidProjectWorkspacePathError(MISSING_FOLDER);
  }
  return folder;
}

async function projectSpace(
  prisma: PrismaClient,
  actor: Pick<Actor, "userId" | "spaceId">,
  name: string,
  folder: string,
): Promise<{ id: string; name: string }> {
  // Applying twice, or onboarding again, reuses the user's own space linked to this folder.
  const existing = await prisma.space.findFirst({
    where: {
      workspacePath: folder,
      deletingAt: null,
      memberships: { some: { userId: actor.userId } },
    },
    select: { id: true, name: true },
    orderBy: { createdAt: "asc" },
  });
  if (existing) return existing;
  const space = await createSpaceForMember(prisma, {
    currentSpaceId: actor.spaceId,
    userId: actor.userId,
    name: name.slice(0, 60),
  });
  await wireProjectWorkspace(prisma, { spaceId: space.id, workspacePath: folder });
  // Like any new project, it starts with its own main agent.
  await ensureMainConversation(prisma, actor.userId, space.id);
  return space;
}

function toRpcError(error: unknown): unknown {
  if (
    error instanceof SpaceLimitError ||
    error instanceof InvalidSpaceNameError ||
    error instanceof InvalidProjectWorkspacePathError ||
    error instanceof InvalidUserFactError
  ) {
    return new ORPCError("BAD_REQUEST", { message: error.message });
  }
  return error;
}

/** Creates a project space per chosen project and saves the chosen facts to the profile. */
export async function applyOnboardingScan(
  prisma: PrismaClient,
  actor: Pick<Actor, "userId" | "spaceId" | "isDeploymentOwner">,
  scanHome: string | undefined,
  input: OnboardingScanApplyInput,
): Promise<OnboardingScanApplyResult> {
  const scan = ownedScan(input.scanId, actor.userId);
  if (scan.status !== "complete" || scan.applying) throw new ORPCError("CONFLICT");
  scan.applying = true;
  try {
    // Ownership or the configured folder can change after the scan; check again before writing.
    if ((await onboardingScanFolder(actor, scanHome)) !== scan.home) {
      throw new ORPCError("FORBIDDEN");
    }
    const projectIds = new Set(input.projectIds);
    const factIds = new Set(input.factIds);
    const projects = scan.projects.filter((project) => projectIds.has(project.id));
    const facts = scan.facts.filter((fact) => factIds.has(fact.id));
    // Every chosen folder is checked before anything is written.
    const chosen = await Promise.all(
      projects.map(async (project) => ({
        name: project.name,
        folder: await confinedProjectFolder(scan.home, project.path),
      })),
    );
    const added =
      facts.length > 0
        ? await updateUserProfile(prisma, actor.userId, {
            addFacts: facts.map(({ category, value }) => ({
              category,
              value,
              source: "onboarding" as const,
            })),
          })
        : [];
    const spaces: Array<{ id: string; name: string }> = [];
    for (const { name, folder } of chosen)
      spaces.push(await projectSpace(prisma, actor, name, folder));
    return { spaces, factsAdded: added.filter((fact) => fact.created).length };
  } catch (error) {
    throw toRpcError(error);
  } finally {
    scan.applying = false;
  }
}