evOo-local/apps/api/src/onboarding-scan.ts

// Onboarding scan — real filesystem scan, Git analysis, and profile compilation.
// Runs synchronously on the host machine (no sandbox needed).
import { spawn } from "node:child_process";
import { access, readdir, stat } from "node:fs/promises";
import { dirname, join } from "node:path";

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

interface ScanResult {
  localRepos: RepoInfo[];
  projectFolders: ProjectFolder[];
  fileSummary: FileSummary;
  profileSummary: ProfileSummary;
  agentBriefing: string;
  createdProjects: CreatedProject[];
  subAgentRoles: SubAgentRole[];
}

interface RepoInfo {
  path: string;
  name: string;
  remote: string | null;
  branch: string;
  lastCommit: string | null;
  languages: Record<string, number>;
  sizeBytes: number;
}

interface ProjectFolder {
  path: string;
  name: string;
  type: "git" | "workspace" | "documents" | "media" | "music" | "code";
  fileCount: number;
  modifiedAt: string;
}

interface FileSummary {
  totalFiles: number;
  totalSizeBytes: number;
  byType: Record<string, number>;
  byExtension: Record<string, number>;
}

interface ProfileSummary {
  interests: string[];
  skills: string[];
  workStyle: string;
  communication: string;
  languages: string[];
  techStack: string[];
}

interface CreatedProject {
  id: string;
  name: string;
  spaceId: string;
  description: string;
}

interface ScanRecord {
  id: string;
  botId: string;
  status: "scanning" | "connecting" | "building" | "briefing" | "complete" | "failed";
  progress: number;
  message: string;
  result: string;
  startedAt: Date;
  completedAt: Date | null;
}

// ---------------------------------------------------------------------------
// Registry (in-memory, 1h TTL)
// ---------------------------------------------------------------------------

const scanRegistry = new Map<string, ScanRecord>();

export async function startOnboardingScan(botId: string, prisma: any): Promise<{ scanId: string }> {
  const scanId = `scan_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
  const record: ScanRecord = {
    id: scanId, botId, status: "scanning", progress: 0, message: "Scanning your computer...",
    result: "[]", startedAt: new Date(), completedAt: null,
  };
  scanRegistry.set(scanId, record);

  // Run async
  (async () => {
    try {
      const homeDir = process.env.HOME ?? "/home/cynar";
      const result = await executeFullScan(homeDir, botId, prisma);
      record.status = "complete";
      record.progress = 100;
      record.message = "Scan complete — profile ready";
      record.result = JSON.stringify(result);
      record.completedAt = new Date();
    } catch (error) {
      record.status = "failed";
      record.message = error instanceof Error ? error.message : "Scan failed";
      record.completedAt = new Date();
    }
  })();

  return { scanId };
}

export async function getScanStatus(scanId: string): Promise<{
  status: ScanRecord["status"]; progress: number; message: string; result?: unknown;
}> {
  const record = scanRegistry.get(scanId);
  if (!record) throw new Error("Scan not found");
  const result = record.result ? JSON.parse(record.result) : undefined;
  return { status: record.status, progress: record.progress, message: record.message, result };
}

// ---------------------------------------------------------------------------
// Full scan pipeline
// ---------------------------------------------------------------------------

async function executeFullScan(homeDir: string, _botId: string, prisma: any): Promise<ScanResult> {
  // Phase 1: Local file scan
  const [fileSummary, projectFolders] = await scanFiles(homeDir);

  // Phase 2: Git repository discovery
  const localRepos = await discoverGitRepos(homeDir);

  // Phase 3: External service analysis (GitHub, etc.)
  // Phase 4: Profile compilation from findings
  const profileSummary = compileProfile(localRepos, fileSummary, projectFolders);

  // Phase 6: Project folder creation
  const createdProjects = await createProjectFolders(localRepos, prisma);

  // Phase 7: Sub-agent role creation
  const subAgentRoles = generateSubAgentRoles(profileSummary, localRepos);

  // Phase 5: Agent briefing (includes sub-agent roles)
  const agentBriefing = generateBriefing(localRepos, profileSummary, fileSummary, subAgentRoles);

  return { localRepos, projectFolders, fileSummary, profileSummary, agentBriefing, createdProjects, subAgentRoles };
}

// ---------------------------------------------------------------------------
// Phase 1: File scan
// ---------------------------------------------------------------------------

async function scanFiles(homeDir: string): Promise<[FileSummary, ProjectFolder[]]> {
  const summary: FileSummary = { totalFiles: 0, totalSizeBytes: 0, byType: {}, byExtension: {} };
  const folders: ProjectFolder[] = [];

  // Scan common directories
  const scanDirs = [
    "Projects", "Documents", "Pictures", "Music", "Downloads", "Desktop",
    "src", "code", "work", "data", "repos", "dev", "workspace",
  ];

  for (const subDir of scanDirs) {
    const dirPath = join(homeDir, subDir);
    try {
      await access(dirPath);
      const stats = await scanDirectory(dirPath, homeDir, subDir);
      summary.totalFiles += stats.totalFiles;
      summary.totalSizeBytes += stats.totalSizeBytes;
      Object.assign(summary.byType, stats.byType);
      Object.assign(summary.byExtension, stats.byExtension);

      if (stats.totalFiles > 0) {
        folders.push({
          path: dirPath,
          name: subDir,
          type: classifyFolderType(subDir),
          fileCount: stats.totalFiles,
          modifiedAt: new Date().toISOString(),
        });
      }
    } catch { /* directory doesn't exist, skip */ }
  }

  return [summary, folders];
}

interface DirStats {
  totalFiles: number;
  totalSizeBytes: number;
  byType: Record<string, number>;
  byExtension: Record<string, number>;
}

async function scanDirectory(dirPath: string, _homeDir: string, _relativePath: string): Promise<DirStats> {
  const stats: DirStats = { totalFiles: 0, totalSizeBytes: 0, byType: {}, byExtension: {} };

  try {
    const entries = await readdir(dirPath, { withFileTypes: true });

    for (const entry of entries) {
      const fullPath = join(dirPath, entry.name);

      if (entry.isDirectory()) {
        // Skip node_modules, .git, etc.
        if (entry.name === "node_modules" || entry.name === ".git" || entry.name === ".venv") continue;
        const subStats = await scanDirectory(fullPath, _homeDir, _relativePath);
        stats.totalFiles += subStats.totalFiles;
        stats.totalSizeBytes += subStats.totalSizeBytes;
        Object.assign(stats.byType, subStats.byType);
        Object.assign(stats.byExtension, subStats.byExtension);
      } else if (entry.isFile()) {
        try {
          const fileStat = await stat(fullPath);
          stats.totalFiles++;
          stats.totalSizeBytes += fileStat.size;

          // Classify by type
          const type = classifyFileType(entry.name);
          stats.byType[type] = (stats.byType[type] || 0) + 1;

          // Classify by extension
          const ext = entry.name.split(".").pop()?.toLowerCase() || "none";
          stats.byExtension[ext] = (stats.byExtension[ext] || 0) + 1;
        } catch { /* permission denied */ }
      }
    }
  } catch { /* permission denied */ }

  return stats;
}

function classifyFileType(filename: string): string {
  const ext = filename.split(".").pop()?.toLowerCase() || "";
  const codeExts = ["ts", "tsx", "js", "jsx", "py", "go", "rs", "java", "c", "cpp", "h", "rb", "php", "swift", "kt"];
  const docExts = ["md", "txt", "doc", "docx", "pdf", "rst", "tex"];
  const imageExts = ["png", "jpg", "jpeg", "gif", "svg", "ico", "webp", "bmp"];
  const musicExts = ["mp3", "wav", "flac", "aac", "ogg", "m4a", "wma"];

  if (codeExts.includes(ext)) return "code";
  if (docExts.includes(ext)) return "documents";
  if (imageExts.includes(ext)) return "images";
  if (musicExts.includes(ext)) return "music";
  if (["json", "yaml", "yml", "toml", "cfg", "ini", "env", "sh", "bash", "zsh"].includes(ext)) return "config";
  return "other";
}

function classifyFolderType(name: string): ProjectFolder["type"] {
  const lower = name.toLowerCase();
  if (lower.includes("music")) return "music";
  if (lower.includes("picture") || lower.includes("image") || lower.includes("photo")) return "media";
  if (lower.includes("document") || lower.includes("doc")) return "documents";
  if (lower.includes("project") || lower.includes("code") || lower.includes("src") || lower.includes("dev") || lower.includes("workspace") || lower.includes("repo")) return "code";
  return "workspace";
}

// ---------------------------------------------------------------------------
// Phase 2: Git repository discovery
// ---------------------------------------------------------------------------

async function discoverGitRepos(homeDir: string): Promise<RepoInfo[]> {
  const repos: RepoInfo[] = [];

  try {
    // Use git to find all repositories under home directory
    const { stdout } = await spawnPromise("git", ["root"], { cwd: homeDir });
    // This only finds the current repo. Instead, search for .git directories
    const gitDirs = await findGitDirectories(homeDir);

    for (const repoDir of gitDirs) {
      try {
        const repo = await analyzeGitRepo(repoDir, homeDir);
        if (repo) repos.push(repo);
      } catch { /* skip broken repos */ }
    }
  } catch { /* git not available */ }

  return repos;
}

async function findGitDirectories(dir: string, depth: number = 0): Promise<string[]> {
  if (depth > 4) return []; // Limit search depth
  const results: string[] = [];

  try {
    const entries = await readdir(dir, { withFileTypes: true });
    for (const entry of entries) {
      const fullPath = join(dir, entry.name);
      if (entry.isDirectory()) {
        if (entry.name === ".git") {
          results.push(dirname(fullPath));
        } else if (entry.name !== "node_modules" && entry.name !== ".git" && entry.name !== ".venv" && entry.name !== "dist") {
          results.push(...await findGitDirectories(fullPath, depth + 1));
        }
      }
    }
  } catch { /* permission denied */ }

  return results;
}

async function analyzeGitRepo(repoDir: string, homeDir: string): Promise<RepoInfo | null> {
  try {
    const name = repoDir.split("/").pop() || "";
    const relPath = repoDir.replace(homeDir + "/", "");

    // Get remote URL
    const remoteResult = await spawnPromise("git", ["remote", "get-url", "origin"], { cwd: repoDir }).catch(() => ({ stdout: "" }));
    const remote = remoteResult.stdout.trim() || null;

    // Get current branch
    const branchResult = await spawnPromise("git", ["branch", "--show-current"], { cwd: repoDir }).catch(() => ({ stdout: "" }));
    const branch = branchResult.stdout.trim() || "unknown";

    // Get last commit
    const commitResult = await spawnPromise("git", ["log", "-1", "--format=%H"], { cwd: repoDir }).catch(() => ({ stdout: "" }));
    const lastCommit = commitResult.stdout.trim() || null;

    // Get language stats via git ls-files + wc
    const languages = await getRepoLanguages(repoDir);

    // Get directory size
    const sizeResult = await spawnPromise("du", ["-sb", repoDir]).catch(() => ({ stdout: "0" }));
    const sizeBytes = parseInt(sizeResult.stdout.split("\t")[0] || "0", 10);

    return { path: repoDir, name, remote, branch, lastCommit, languages, sizeBytes };
  } catch {
    return null;
  }
}

async function getRepoLanguages(repoDir: string): Promise<Record<string, number>> {
  const langs: Record<string, number> = {};
  try {
    const { stdout } = await spawnPromise("git", ["ls-files"], { cwd: repoDir }).catch(() => ({ stdout: "" }));
    const files = stdout.trim().split("\n").filter(Boolean);
    for (const file of files) {
      const ext = file.split(".").pop()?.toLowerCase() || "";
      const langMap: Record<string, string> = {
        ts: "TypeScript", tsx: "TypeScript", js: "JavaScript", jsx: "JavaScript",
        py: "Python", go: "Go", rs: "Rust", java: "Java", c: "C", cpp: "C++",
        rb: "Ruby", php: "PHP", swift: "Swift", kt: "Kotlin",
      };
      const lang = langMap[ext] || ext || "unknown";
      langs[lang] = (langs[lang] || 0) + 1;
    }
  } catch { /* no languages */ }
  return langs;
}

// ---------------------------------------------------------------------------
// Phase 3: Profile compilation
// ---------------------------------------------------------------------------

function compileProfile(repos: RepoInfo[], fileSummary: FileSummary, folders: ProjectFolder[]): ProfileSummary {
  const interests: string[] = [];
  const skills: string[] = [];
  const techStack: string[] = [];
  const languageSet = new Set<string>();

  // Extract tech stack from repos
  for (const repo of repos) {
    for (const lang of Object.keys(repo.languages)) {
      techStack.push(lang);
      languageSet.add(lang);
    }
  }

  // Extract interests from file types
  for (const [type, count] of Object.entries(fileSummary.byType)) {
    if (count > 5) {
      interests.push(`${type} (${count} files)`);
    }
  }

  // Derive skills from languages
  const langToSkill: Record<string, string> = {
    "TypeScript": "TypeScript/JavaScript", Python: "Python", Go: "Go",
    "Rust": "Rust", Java: "Java", C: "C/C++", "C++": "C/C++", Ruby: "Ruby",
    "Swift": "Swift/Kotlin", PHP: "PHP",
  };
  for (const lang of techStack) {
    const skill = langToSkill[lang];
    if (skill && !skills.includes(skill)) skills.push(skill);
  }

  // Deduplicate
  const uniqueInterests = [...new Set(interests)].slice(0, 10);
  const uniqueSkills = [...new Set(skills)].slice(0, 15);
  const uniqueTech = [...new Set(techStack)].sort();

  return {
    interests: uniqueInterests,
    skills: uniqueSkills,
    workStyle: uniqueTech.length > 3 ? "multi-language" : uniqueTech[0] ? `${uniqueTech[0]}-focused` : "general",
    communication: "structured",
    languages: Array.from(languageSet),
    techStack: uniqueTech,
  };
}

// ---------------------------------------------------------------------------
// Phase 4: Agent briefing
// ---------------------------------------------------------------------------

function generateBriefing(
  repos: RepoInfo[],
  profile: ProfileSummary,
  fileSummary: FileSummary,
  subAgentRoles: SubAgentRole[],
): string {
  const lines: string[] = [];
  lines.push(`# User Profile Briefing`);
  lines.push(``);
  lines.push(`## Technical Profile`);
  lines.push(`- **Tech Stack:** ${profile.techStack.join(", ") || "N/A"}`);
  lines.push(`- **Languages:** ${profile.languages.join(", ") || "N/A"}`);
  lines.push(`- **Work Style:** ${profile.workStyle}`);
  lines.push(``);
  lines.push(`## Local Repositories (${repos.length} found)`);
  for (const repo of repos) {
    lines.push(`- **${repo.name}** — ${repo.branch}, ${Object.values(repo.languages).reduce((a, b) => a + b, 0)} files, ${repo.remote || "no remote"}`);
  }
  lines.push(``);
  lines.push(`## File System Summary`);
  lines.push(`- **Total Files:** ${fileSummary.totalFiles.toLocaleString()}`);
  lines.push(`- **Total Size:** ${(fileSummary.totalSizeBytes / (1024 * 1024)).toFixed(0)} MB`);
  lines.push(`- **Categories:** ${Object.entries(fileSummary.byType).map(([t, c]) => `${t}(${c})`).join(", ")}`);
  lines.push(``);
  lines.push(`## User Interests`);
  lines.push(`- ${profile.interests.slice(0, 5).join("\n- ")}`);
  lines.push(``);
  lines.push(`## User Skills`);
  lines.push(`- ${profile.skills.slice(0, 5).join("\n- ")}`);
  lines.push(``);
  lines.push(`## Assigned Specialist Roles`);
  for (const role of subAgentRoles) {
    lines.push(`### ${role.name}`);
    lines.push(`- **Role:** ${role.role}`);
    lines.push(`- **Description:** ${role.description}`);
    lines.push(`- **Instructions:** ${role.instructions}`);
    lines.push(`- **Skills:** ${role.skills.join(", ")}`);
    lines.push("");
  }
  lines.push(`---`);
  lines.push(`This profile should be used to tailor all future interactions, project recommendations, and agent assignments.`);
  return lines.join("\n");
}

// ---------------------------------------------------------------------------
// Phase 5: Project folder creation
// ---------------------------------------------------------------------------

async function createProjectFolders(repos: RepoInfo[], prisma: any): Promise<CreatedProject[]> {
  const created: CreatedProject[] = [];

  for (const repo of repos) {
    try {
      // Check if space already exists
      const existing = await prisma.space.findFirst({ where: { title: repo.name } });
      if (existing) {
        created.push({ id: existing.id, name: repo.name, spaceId: existing.id, description: repo.remote || repo.path });
        continue;
      }
      // Create new space
      const space = await prisma.space.create({
        data: { title: repo.name, path: repo.path },
      });
      created.push({ id: space.id, name: repo.name, spaceId: space.id, description: repo.remote || repo.path });
    } catch {
      // Space might already exist or DB unavailable — skip
    }
  }

  return created;
}

// ---------------------------------------------------------------------------
// Phase 6: Sub-agent role creation
// ---------------------------------------------------------------------------

interface SubAgentRole {
  name: string;
  role: string;
  description: string;
  instructions: string;
  skills: string[];
}

function generateSubAgentRoles(profile: ProfileSummary, repos: RepoInfo[]): SubAgentRole[] {
  const roles: SubAgentRole[] = [];
  const hasCode = profile.languages?.length > 0;
  const hasDesign = profile.interests?.some(i => i.toLowerCase().includes("design") || i.toLowerCase().includes("ui"));
  const hasData = profile.interests?.some(i => i.toLowerCase().includes("data") || i.toLowerCase().includes("analytics") || i.toLowerCase().includes("database"));
  const hasDevOps = profile.interests?.some(i => i.toLowerCase().includes("infra") || i.toLowerCase().includes("ops") || i.toLowerCase().includes("deploy"));
  const hasMusic = profile.interests?.some(i => i.toLowerCase().includes("music"));
  const hasWriting = profile.interests?.some(i => i.toLowerCase().includes("write") || i.toLowerCase().includes("blog"));

  if (hasCode) {
    const primaryLang = profile.languages?.[0] || "TypeScript";
    roles.push({
      name: "Code Engineer",
      role: "software-engineer",
      description: `Specializes in ${primaryLang} development and code review`,
      instructions: `You are the Code Engineer specialist. Your expertise is in ${profile.languages?.join(", ") || "software development"}. Review code, write tests, debug issues, and implement features. You have access to the user's local repositories and can run commands to build and test code.`,
      skills: profile.skills.filter(s => s.toLowerCase().includes("code") || s.toLowerCase().includes("program")),
    });
  }

  if (hasDesign) {
    roles.push({
      name: "UI/UX Designer",
      role: "ui-designer",
      description: "Creates polished interfaces and design systems",
      instructions: "You are the UI/UX Designer specialist. Focus on creating beautiful, accessible interfaces. Review designs, suggest improvements, and ensure consistency across the user's projects.",
      skills: ["design", "UI/UX"],
    });
  }

  if (hasData) {
    roles.push({
      name: "Data Analyst",
      role: "data-analyst",
      description: "Analyzes data, creates reports, and extracts insights",
      instructions: "You are the Data Analyst specialist. Process data from the user's projects, create visualizations, generate reports, and provide actionable insights.",
      skills: ["data analysis", "statistics"],
    });
  }

  if (hasDevOps) {
    roles.push({
      name: "DevOps Engineer",
      role: "devops-engineer",
      description: "Manages infrastructure, deployments, and CI/CD",
      instructions: "You are the DevOps Engineer specialist. Handle deployments, manage infrastructure, configure CI/CD pipelines, and ensure system reliability.",
      skills: ["DevOps", "infrastructure"],
    });
  }

  if (hasMusic) {
    roles.push({
      name: "Music Producer",
      role: "music-producer",
      description: "Assists with music production and audio engineering",
      instructions: "You are the Music Producer specialist. Help with music composition, audio engineering, mixing, and mastering. You understand the user's music projects and can provide expert advice.",
      skills: ["music production", "audio engineering"],
    });
  }

  if (hasWriting) {
    roles.push({
      name: "Content Writer",
      role: "content-writer",
      description: "Creates and edits written content",
      instructions: "You are the Content Writer specialist. Write blog posts, documentation, articles, and other content for the user's projects and personal use.",
      skills: ["writing", "content creation"],
    });
  }

  // Always include a general assistant
  roles.unshift({
    name: "General Assistant",
    role: "general-assistant",
    description: "Handles general tasks and coordinates other specialists",
    instructions: `You are the General Assistant and team coordinator. Your role is to handle general queries, coordinate with other specialists (${roles.slice(1).map(r => r.name).join(", ") || "none yet"}), and manage the user's daily workflow. You have access to all tools and can delegate tasks to specialists when needed.`,
    skills: ["task management", "coordination"],
  });

  return roles;
}

// ---------------------------------------------------------------------------
// Utility
// ---------------------------------------------------------------------------

function spawnPromise(cmd: string, args: string[], options?: { cwd?: string }): Promise<{ stdout: string; stderr: string }> {
  return new Promise((resolve, reject) => {
    const child = spawn(cmd, args, { ...options, timeout: 15000 });
    let stdout = "";
    let stderr = "";
    child.stdout?.on("data", (d: Buffer) => { stdout += d.toString(); });
    child.stderr?.on("data", (d: Buffer) => { stderr += d.toString(); });
    child.on("error", () => reject(new Error("spawn failed")));
    child.on("close", (code) => {
      if (code === 0 || code === null) resolve({ stdout, stderr });
      else reject(new Error(`exit ${code}`));
    });
  });
}

// Keep old registry cleanup
setInterval(() => {
  const cutoff = Date.now() - 3600000;
  for (const [id, r] of scanRegistry) {
    if (r.completedAt && r.completedAt.getTime() < cutoff) scanRegistry.delete(id);
  }
}, 60000);