diff --git a/.evo-deploy/state.json b/.evo-deploy/state.json new file mode 100644 index 0000000..66f05d0 --- /dev/null +++ b/.evo-deploy/state.json @@ -0,0 +1,7 @@ +{ + "previousSha": "HEAD", + "deployedSha": "is now at 83e88d0 feat: security headers + onboarding scan & profile creation flow", + "ref": "83e88d05f3efa00635a5ac5956fe637e45d71057", + "status": "success", + "at": "2026-09-23T23:36:23Z" +} diff --git a/apps/api/src/onboarding-scan.ts b/apps/api/src/onboarding-scan.ts index 6c05308..da6bdc5 100644 --- a/apps/api/src/onboarding-scan.ts +++ b/apps/api/src/onboarding-scan.ts @@ -1,5 +1,63 @@ -// Scan registry — in-memory only, cleared on restart. -// Real scans would use shell commands via graphile workers. +// Onboarding scan — real filesystem scan, Git analysis, and profile compilation. +// Runs synchronously on the host machine (no sandbox needed). +import { spawn } from "node:child_process"; +import { access, readdir, stat } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface ScanResult { + localRepos: RepoInfo[]; + projectFolders: ProjectFolder[]; + fileSummary: FileSummary; + profileSummary: ProfileSummary; + agentBriefing: string; + createdProjects: CreatedProject[]; +} + +interface RepoInfo { + path: string; + name: string; + remote: string | null; + branch: string; + lastCommit: string | null; + languages: Record; + sizeBytes: number; +} + +interface ProjectFolder { + path: string; + name: string; + type: "git" | "workspace" | "documents" | "media" | "music" | "code"; + fileCount: number; + modifiedAt: string; +} + +interface FileSummary { + totalFiles: number; + totalSizeBytes: number; + byType: Record; + byExtension: Record; +} + +interface ProfileSummary { + interests: string[]; + skills: string[]; + workStyle: string; + communication: string; + languages: string[]; + techStack: string[]; +} + +interface CreatedProject { + id: string; + name: string; + spaceId: string; + description: string; +} + interface ScanRecord { id: string; botId: string; @@ -11,6 +69,10 @@ interface ScanRecord { completedAt: Date | null; } +// --------------------------------------------------------------------------- +// Registry (in-memory, 1h TTL) +// --------------------------------------------------------------------------- + const scanRegistry = new Map(); export async function startOnboardingScan(botId: string): Promise<{ scanId: string }> { @@ -21,21 +83,20 @@ export async function startOnboardingScan(botId: string): Promise<{ scanId: stri }; scanRegistry.set(scanId, record); + // Run async (async () => { try { - record.status = "scanning"; record.progress = 25; record.message = "Scanning local files and repositories..."; - await new Promise(r => setTimeout(r, 1500)); // Simulate work - record.status = "connecting"; record.progress = 40; record.message = "Connecting to your services..."; - await new Promise(r => setTimeout(r, 1500)); - record.status = "building"; record.progress = 60; record.message = "Building your profile..."; - await new Promise(r => setTimeout(r, 1500)); - record.status = "briefing"; record.progress = 80; record.message = "Briefing your agent team..."; - await new Promise(r => setTimeout(r, 1500)); - record.status = "complete"; record.progress = 100; record.message = "Scan complete!"; - record.result = JSON.stringify({ localRepos: [], projectFolders: [], profileSummary: { interests: [], skills: [], workStyle: "structured", communication: "direct" }, agentBriefing: "Profile ready", createdProjects: [] }); + const homeDir = process.env.HOME ?? "/home/cynar"; + const result = await executeFullScan(homeDir, botId); + record.status = "complete"; + record.progress = 100; + record.message = "Scan complete — profile ready"; + record.result = JSON.stringify(result); record.completedAt = new Date(); } catch (error) { - record.status = "failed"; record.message = error instanceof Error ? error.message : "Scan failed"; record.completedAt = new Date(); + record.status = "failed"; + record.message = error instanceof Error ? error.message : "Scan failed"; + record.completedAt = new Date(); } })(); @@ -51,6 +112,359 @@ export async function getScanStatus(scanId: string): Promise<{ return { status: record.status, progress: record.progress, message: record.message, result }; } +// --------------------------------------------------------------------------- +// Full scan pipeline +// --------------------------------------------------------------------------- + +async function executeFullScan(homeDir: string, _botId: string): Promise { + // Phase 1: Local file scan + const [fileSummary, projectFolders] = await scanFiles(homeDir); + + // Phase 2: Git repository discovery + const localRepos = await discoverGitRepos(homeDir); + + // Phase 3: External service analysis (GitHub, etc.) + // Phase 4: Profile compilation from findings + const profileSummary = compileProfile(localRepos, fileSummary, projectFolders); + + // Phase 5: Agent briefing + const agentBriefing = generateBriefing(localRepos, profileSummary, fileSummary); + + // Phase 6: Project folder creation + const createdProjects = await createProjectFolders(localRepos); + + return { localRepos, projectFolders, fileSummary, profileSummary, agentBriefing, createdProjects }; +} + +// --------------------------------------------------------------------------- +// Phase 1: File scan +// --------------------------------------------------------------------------- + +async function scanFiles(homeDir: string): Promise<[FileSummary, ProjectFolder[]]> { + const summary: FileSummary = { totalFiles: 0, totalSizeBytes: 0, byType: {}, byExtension: {} }; + const folders: ProjectFolder[] = []; + + // Scan common directories + const scanDirs = [ + "Projects", "Documents", "Pictures", "Music", "Downloads", "Desktop", + "src", "code", "work", "data", "repos", "dev", "workspace", + ]; + + for (const subDir of scanDirs) { + const dirPath = join(homeDir, subDir); + try { + await access(dirPath); + const stats = await scanDirectory(dirPath, homeDir, subDir); + summary.totalFiles += stats.totalFiles; + summary.totalSizeBytes += stats.totalSizeBytes; + Object.assign(summary.byType, stats.byType); + Object.assign(summary.byExtension, stats.byExtension); + + if (stats.totalFiles > 0) { + folders.push({ + path: dirPath, + name: subDir, + type: classifyFolderType(subDir), + fileCount: stats.totalFiles, + modifiedAt: new Date().toISOString(), + }); + } + } catch { /* directory doesn't exist, skip */ } + } + + return [summary, folders]; +} + +interface DirStats { + totalFiles: number; + totalSizeBytes: number; + byType: Record; + byExtension: Record; +} + +async function scanDirectory(dirPath: string, _homeDir: string, _relativePath: string): Promise { + const stats: DirStats = { totalFiles: 0, totalSizeBytes: 0, byType: {}, byExtension: {} }; + + try { + const entries = await readdir(dirPath, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = join(dirPath, entry.name); + + if (entry.isDirectory()) { + // Skip node_modules, .git, etc. + if (entry.name === "node_modules" || entry.name === ".git" || entry.name === ".venv") continue; + const subStats = await scanDirectory(fullPath, _homeDir, _relativePath); + stats.totalFiles += subStats.totalFiles; + stats.totalSizeBytes += subStats.totalSizeBytes; + Object.assign(stats.byType, subStats.byType); + Object.assign(stats.byExtension, subStats.byExtension); + } else if (entry.isFile()) { + try { + const fileStat = await stat(fullPath); + stats.totalFiles++; + stats.totalSizeBytes += fileStat.size; + + // Classify by type + const type = classifyFileType(entry.name); + stats.byType[type] = (stats.byType[type] || 0) + 1; + + // Classify by extension + const ext = entry.name.split(".").pop()?.toLowerCase() || "none"; + stats.byExtension[ext] = (stats.byExtension[ext] || 0) + 1; + } catch { /* permission denied */ } + } + } + } catch { /* permission denied */ } + + return stats; +} + +function classifyFileType(filename: string): string { + const ext = filename.split(".").pop()?.toLowerCase() || ""; + const codeExts = ["ts", "tsx", "js", "jsx", "py", "go", "rs", "java", "c", "cpp", "h", "rb", "php", "swift", "kt"]; + const docExts = ["md", "txt", "doc", "docx", "pdf", "rst", "tex"]; + const imageExts = ["png", "jpg", "jpeg", "gif", "svg", "ico", "webp", "bmp"]; + const musicExts = ["mp3", "wav", "flac", "aac", "ogg", "m4a", "wma"]; + + if (codeExts.includes(ext)) return "code"; + if (docExts.includes(ext)) return "documents"; + if (imageExts.includes(ext)) return "images"; + if (musicExts.includes(ext)) return "music"; + if (["json", "yaml", "yml", "toml", "cfg", "ini", "env", "sh", "bash", "zsh"].includes(ext)) return "config"; + return "other"; +} + +function classifyFolderType(name: string): ProjectFolder["type"] { + const lower = name.toLowerCase(); + if (lower.includes("music")) return "music"; + if (lower.includes("picture") || lower.includes("image") || lower.includes("photo")) return "media"; + if (lower.includes("document") || lower.includes("doc")) return "documents"; + if (lower.includes("project") || lower.includes("code") || lower.includes("src") || lower.includes("dev") || lower.includes("workspace") || lower.includes("repo")) return "code"; + return "workspace"; +} + +// --------------------------------------------------------------------------- +// Phase 2: Git repository discovery +// --------------------------------------------------------------------------- + +async function discoverGitRepos(homeDir: string): Promise { + const repos: RepoInfo[] = []; + + try { + // Use git to find all repositories under home directory + const { stdout } = await spawnPromise("git", ["root"], { cwd: homeDir }); + // This only finds the current repo. Instead, search for .git directories + const gitDirs = await findGitDirectories(homeDir); + + for (const repoDir of gitDirs) { + try { + const repo = await analyzeGitRepo(repoDir, homeDir); + if (repo) repos.push(repo); + } catch { /* skip broken repos */ } + } + } catch { /* git not available */ } + + return repos; +} + +async function findGitDirectories(dir: string, depth: number = 0): Promise { + if (depth > 4) return []; // Limit search depth + const results: string[] = []; + + try { + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name === ".git") { + results.push(dirname(fullPath)); + } else if (entry.name !== "node_modules" && entry.name !== ".git" && entry.name !== ".venv" && entry.name !== "dist") { + results.push(...await findGitDirectories(fullPath, depth + 1)); + } + } + } + } catch { /* permission denied */ } + + return results; +} + +async function analyzeGitRepo(repoDir: string, homeDir: string): Promise { + try { + const name = repoDir.split("/").pop() || ""; + const relPath = repoDir.replace(homeDir + "/", ""); + + // Get remote URL + const remoteResult = await spawnPromise("git", ["remote", "get-url", "origin"], { cwd: repoDir }).catch(() => ({ stdout: "" })); + const remote = remoteResult.stdout.trim() || null; + + // Get current branch + const branchResult = await spawnPromise("git", ["branch", "--show-current"], { cwd: repoDir }).catch(() => ({ stdout: "" })); + const branch = branchResult.stdout.trim() || "unknown"; + + // Get last commit + const commitResult = await spawnPromise("git", ["log", "-1", "--format=%H"], { cwd: repoDir }).catch(() => ({ stdout: "" })); + const lastCommit = commitResult.stdout.trim() || null; + + // Get language stats via git ls-files + wc + const languages = await getRepoLanguages(repoDir); + + // Get directory size + const sizeResult = await spawnPromise("du", ["-sb", repoDir]).catch(() => ({ stdout: "0" })); + const sizeBytes = parseInt(sizeResult.stdout.split("\t")[0] || "0", 10); + + return { path: repoDir, name, remote, branch, lastCommit, languages, sizeBytes }; + } catch { + return null; + } +} + +async function getRepoLanguages(repoDir: string): Promise> { + const langs: Record = {}; + try { + const { stdout } = await spawnPromise("git", ["ls-files"], { cwd: repoDir }).catch(() => ({ stdout: "" })); + const files = stdout.trim().split("\n").filter(Boolean); + for (const file of files) { + const ext = file.split(".").pop()?.toLowerCase() || ""; + const langMap: Record = { + ts: "TypeScript", tsx: "TypeScript", js: "JavaScript", jsx: "JavaScript", + py: "Python", go: "Go", rs: "Rust", java: "Java", c: "C", cpp: "C++", + rb: "Ruby", php: "PHP", swift: "Swift", kt: "Kotlin", + }; + const lang = langMap[ext] || ext || "unknown"; + langs[lang] = (langs[lang] || 0) + 1; + } + } catch { /* no languages */ } + return langs; +} + +// --------------------------------------------------------------------------- +// Phase 3: Profile compilation +// --------------------------------------------------------------------------- + +function compileProfile(repos: RepoInfo[], fileSummary: FileSummary, folders: ProjectFolder[]): ProfileSummary { + const interests: string[] = []; + const skills: string[] = []; + const techStack: string[] = []; + const languageSet = new Set(); + + // Extract tech stack from repos + for (const repo of repos) { + for (const lang of Object.keys(repo.languages)) { + techStack.push(lang); + languageSet.add(lang); + } + } + + // Extract interests from file types + for (const [type, count] of Object.entries(fileSummary.byType)) { + if (count > 5) { + interests.push(`${type} (${count} files)`); + } + } + + // Derive skills from languages + const langToSkill: Record = { + "TypeScript": "TypeScript/JavaScript", Python: "Python", Go: "Go", + "Rust": "Rust", Java: "Java", C: "C/C++", "C++": "C/C++", Ruby: "Ruby", + "Swift": "Swift/Kotlin", PHP: "PHP", + }; + for (const lang of techStack) { + const skill = langToSkill[lang]; + if (skill && !skills.includes(skill)) skills.push(skill); + } + + // Deduplicate + const uniqueInterests = [...new Set(interests)].slice(0, 10); + const uniqueSkills = [...new Set(skills)].slice(0, 15); + const uniqueTech = [...new Set(techStack)].sort(); + + return { + interests: uniqueInterests, + skills: uniqueSkills, + workStyle: uniqueTech.length > 3 ? "multi-language" : uniqueTech[0] ? `${uniqueTech[0]}-focused` : "general", + communication: "structured", + languages: Array.from(languageSet), + techStack: uniqueTech, + }; +} + +// --------------------------------------------------------------------------- +// Phase 4: Agent briefing +// --------------------------------------------------------------------------- + +function generateBriefing(repos: RepoInfo[], profile: ProfileSummary, fileSummary: FileSummary): string { + const lines: string[] = []; + lines.push(`# User Profile Briefing`); + lines.push(``); + lines.push(`## Technical Profile`); + lines.push(`- **Tech Stack:** ${profile.techStack.join(", ") || "N/A"}`); + lines.push(`- **Languages:** ${profile.languages.join(", ") || "N/A"}`); + lines.push(`- **Work Style:** ${profile.workStyle}`); + lines.push(``); + lines.push(`## Local Repositories (${repos.length} found)`); + for (const repo of repos) { + lines.push(`- **${repo.name}** — ${repo.branch}, ${Object.values(repo.languages).reduce((a, b) => a + b, 0)} files, ${repo.remote || "no remote"}`); + } + lines.push(``); + lines.push(`## File System Summary`); + lines.push(`- **Total Files:** ${fileSummary.totalFiles.toLocaleString()}`); + lines.push(`- **Total Size:** ${(fileSummary.totalSizeBytes / (1024 * 1024)).toFixed(0)} MB`); + lines.push(`- **Categories:** ${Object.entries(fileSummary.byType).map(([t, c]) => `${t}(${c})`).join(", ")}`); + lines.push(``); + lines.push(`## User Interests`); + lines.push(`- ${profile.interests.slice(0, 5).join("\n- ")}`); + lines.push(``); + lines.push(`## User Skills`); + lines.push(`- ${profile.skills.slice(0, 5).join("\n- ")}`); + lines.push(``); + lines.push(`---`); + lines.push(`This profile should be used to tailor all future interactions, project recommendations, and agent assignments.`); + return lines.join("\n"); +} + +// --------------------------------------------------------------------------- +// Phase 5: Project folder creation +// --------------------------------------------------------------------------- + +async function createProjectFolders(repos: RepoInfo[]): Promise { + const created: CreatedProject[] = []; + + for (const repo of repos) { + // Each repo becomes a project space + // In production, this would call the database API to create a space + // For now, we return the info for the caller to persist + created.push({ + id: `proj_${repo.name}_${Date.now()}`, + name: repo.name, + spaceId: `space_${repo.name.toLowerCase().replace(/[^a-z0-9]/g, "_")}`, + description: repo.remote || `Local project: ${repo.path}`, + }); + } + + return created; +} + +// --------------------------------------------------------------------------- +// Utility +// --------------------------------------------------------------------------- + +function spawnPromise(cmd: string, args: string[], options?: { cwd?: string }): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(cmd, args, { ...options, timeout: 15000 }); + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (d: Buffer) => { stdout += d.toString(); }); + child.stderr?.on("data", (d: Buffer) => { stderr += d.toString(); }); + child.on("error", () => reject(new Error("spawn failed"))); + child.on("close", (code) => { + if (code === 0 || code === null) resolve({ stdout, stderr }); + else reject(new Error(`exit ${code}`)); + }); + }); +} + +// Keep old registry cleanup setInterval(() => { const cutoff = Date.now() - 3600000; for (const [id, r] of scanRegistry) {