import type { JobPublisher, JobWorkerHost } from "@rakazo/adapter-kit"; import { ComposioConnector, IntegrationProviderSettings } from "@rakazo/adapters"; import { loadRootEnv } from "@rakazo/core/node/load-root-env"; loadRootEnv(); import { ChatSdkMessagingSurface, createBackgroundJobHandlers, createCloudAgentConnection, createConnectorStack, createJobReconciler, createMessagingContextLoader, createPostgresReconciliationLeadership, createMediaJobHandlers, createMediaServices, createProfileConsolidationScheduler, createRunExecutor, createRunSandbox, createRunSecretWriter, createWebProvider, databaseCapacityBackoffMs, EncryptedSecretStore, ExpoPushProvider, GraphileJobPublisher, GraphileJobWorkerHost, InMemoryJobQueue, InstalledConnectorProvider, isComposioEnabled, isMessagingSurfaceEnabled, isPipedreamEnabled, LocalAgentHomeStore, LocalArtifactStore, mediaConfigFromEnv, McpConnector, McpOAuthBroker, messagingEnvFromProcess, messagingPlatformsFromEnv, PiAgentRuntime, PipedreamConnector, PostgresRealtimeFanout, pipedreamConfigFromEnv, reconcileCloudAgents, reconcileComputerUpdates, reconcileDelegatedTasks, resolveDeploymentModel, resolvePiSessionRoot, resolveSandboxProvider, ScriptedAgentRuntime, SpaceMemoryProviderResolver, } from "@rakazo/adapters"; import { resolveEncryptionKey, resolveSupervisorToken } from "@rakazo/core"; import { createDb, createThreadEvents, isTooManyDatabaseConnections, parsePositiveInteger, } from "@rakazo/db"; import { SERVICE_NAMES } from "@rakazo/logging"; import { createRootLogger } from "@rakazo/logging/axiom"; import { MarkdownMemoryStore } from "@rakazo/memory"; const logger = createRootLogger(SERVICE_NAMES.worker); async function main() { const databaseUrl = process.env.DATABASE_URL; if (!databaseUrl) throw new Error("DATABASE_URL is required"); // Shared by Prisma, the reconciliation leadership lock, and both graphile-worker // components (see GraphileJobPublisher/GraphileJobWorkerHost) — one pool instead // of four separate ones. Keep this modest: graphile holds a LISTEN client and // leadership holds an advisory-lock client for the process lifetime, and a // larger max just competes for Postgres max_connections (53300). const { prisma, pool } = createDb(databaseUrl, { poolMax: parsePositiveInteger(process.env.DB_POOL_MAX, 8), applicationName: "rakazo-worker", }); const realtime = new PostgresRealtimeFanout({ connectionString: process.env.REALTIME_DATABASE_URL ?? databaseUrl, publisher: pool, }); const secrets = new EncryptedSecretStore(resolveEncryptionKey(process.env)); const events = createThreadEvents(prisma, realtime, { runSecretWriter: createRunSecretWriter(secrets), }); const dataDir = process.env.DATA_DIR ?? "./data"; const runtime = process.env.AGENT_RUNTIME === "scripted" ? new ScriptedAgentRuntime() : new PiAgentRuntime({ sessionRoot: resolvePiSessionRoot(dataDir) }); // Same resolver the API uses, so both processes agree on provider, model and key. const { key: deploymentModelKey } = resolveDeploymentModel(); const sandboxProvider = resolveSandboxProvider(process.env); const sandbox = createRunSandbox(sandboxProvider, { supervisorUrl: process.env.SANDBOX_SUPERVISOR_URL ?? "http://127.0.0.1:7091", supervisorToken: sandboxProvider === "docker" ? resolveSupervisorToken(process.env) : undefined, e2bApiKey: process.env.E2B_API_KEY, daytonaApiKey: process.env.DAYTONA_API_KEY, daytonaApiUrl: process.env.DAYTONA_API_URL, daytonaTarget: process.env.DAYTONA_TARGET, boxApiKey: process.env.BOX_API_KEY, boxApiUrl: process.env.BOX_API_URL ?? process.env.BOX_BASE_URL, dataDir, prisma, }); const mcpOAuth = new McpOAuthBroker(prisma, secrets); const mcp = new McpConnector( prisma, secrets, { stdioEnabled: process.env.MCP_STDIO_ENABLED === "true", allowedCommands: (process.env.MCP_STDIO_ALLOWED_COMMANDS ?? "") .split(",") .map((v) => v.trim()) .filter(Boolean), }, mcpOAuth, ); const pipedreamConfig = pipedreamConfigFromEnv({ pipedreamClientId: process.env.PIPEDREAM_CLIENT_ID, pipedreamClientSecret: process.env.PIPEDREAM_CLIENT_SECRET, pipedreamProjectId: process.env.PIPEDREAM_PROJECT_ID, pipedreamEnvironment: process.env.PIPEDREAM_ENVIRONMENT, encryptionKey: resolveEncryptionKey(process.env), }); const pipedream = isPipedreamEnabled(pipedreamConfig) ? new PipedreamConnector(pipedreamConfig) : undefined; // pollInboundMessages stays false (the default) here: this process // only ever sends outbound (messaging.deliver jobs). It must never poll // Telegram — that would steal the single getUpdates slot away from the // API process, which is the one with the inbound sink actually wired up. const messagingPlatforms = messagingPlatformsFromEnv(messagingEnvFromProcess(process.env)); const messaging = isMessagingSurfaceEnabled(messagingPlatforms, { deploymentModelKey, openSignup: process.env.MESSAGING_OPEN_SIGNUP === "true", }) ? new ChatSdkMessagingSurface(messagingPlatforms) : undefined; const integrationSettings = new IntegrationProviderSettings( prisma, secrets, resolveEncryptionKey(process.env), { composio: isComposioEnabled(process.env.COMPOSIO_API_KEY) ? new ComposioConnector(process.env.COMPOSIO_API_KEY) : undefined, pipedream, }, ); const stack = createConnectorStack(false, undefined, [ new InstalledConnectorProvider(prisma, secrets), ...integrationSettings.providers(), mcp, ]); const connector = stack.destination; await connector.start(); integrationSettings.warmDirectories(); const memoryProviders = new SpaceMemoryProviderResolver(prisma, secrets); const home = new LocalAgentHomeStore(dataDir); const artifacts = new LocalArtifactStore(dataDir); const inMemoryJobs = process.env.WAKEUP_DRIVER === "memory" ? new InMemoryJobQueue() : undefined; const jobs: JobPublisher = inMemoryJobs ?? new GraphileJobPublisher(pool); const jobHost: JobWorkerHost = inMemoryJobs ?? new GraphileJobWorkerHost(pool, { concurrency: parsePositiveInteger(process.env.GRAPHILE_WORKER_CONCURRENCY, 4), }); // One provider instance so emulator launches and polls share the same Map. const cloudAgent = createCloudAgentConnection(); const executor = createRunExecutor({ prisma, runtime, sandbox, memory: new MarkdownMemoryStore(prisma), memoryProviders, home, artifacts, connector: stack.connector, connectors: stack.connector, listConnectedPluginSlugs: async (userId) => { const provider = await integrationSettings.resolve("composio"); if (!provider) return []; return provider.listConnectedExternalIds({ userId, spaceId: "", operationId: "connections.sync", traceId: "connections.sync", signal: AbortSignal.timeout(15_000), }); }, secrets: [ deploymentModelKey ?? "", process.env.COMPOSIO_API_KEY ?? "", process.env.CURSOR_API_KEY ?? "", ].filter(Boolean), secretStore: secrets, deploymentModelKey, dataDir, notifications: new ExpoPushProvider(dataDir), jobs, events, messaging: messaging ? createMessagingContextLoader(prisma) : undefined, web: createWebProvider(), cloudAgent, }); const mediaAbort = new AbortController(); const mediaServices = createMediaServices({ config: mediaConfigFromEnv(process.env), prisma, jobs, artifacts, runtime, realtime, resolveModel: (scope) => executor.resolveModel(scope), }); const mediaJobs = mediaServices ? createMediaJobHandlers(mediaServices, { signal: mediaAbort.signal, limits: { gpu: parsePositiveInteger(process.env.MEDIA_GPU_CONCURRENCY, 1), cpu: parsePositiveInteger(process.env.MEDIA_CPU_CONCURRENCY, 2), }, }) : null; const jobHandlers = createBackgroundJobHandlers({ executor, prisma, sandbox, home, jobs, events, ...(mediaJobs ? { media: mediaJobs.handlers } : {}), workerId: process.pid.toString(), runtime, secretStore: secrets, memoryProviders, deploymentModelKey, messaging, cloudAgent, }); // graphile-worker run() connects through the shared pool. createPool already // retries connect() on 53300 a finite number of times. Keep retrying start // until Postgres has capacity: exhausting then returning from main().catch // left a live process that held connections but never ran jobs or registered // signal handlers, even after capacity returned. Do not exit(1) here; that // crash-loops into the same saturated Postgres. GraphileJobWorkerHost also // observes runner.promise after start and restarts with the same backoff if // the runner dies later on 53300 (unhandledRejection still swallows that // code so we do not Docker crash-loop on transient completeJob failures). for (let attempt = 0; ; attempt += 1) { try { await jobHost.start(jobHandlers); break; } catch (error) { if (!isTooManyDatabaseConnections(error)) throw error; logger.error("worker job host start waiting on database capacity", error); await new Promise((resolve) => setTimeout(resolve, databaseCapacityBackoffMs(attempt))); } } if (mediaJobs) { void mediaJobs.resume().catch((error) => logger.error("media ingest resume failed", error)); } const reconciler = createJobReconciler({ prisma, jobs, events, leadership: createPostgresReconciliationLeadership(pool), reconcileCloudAgents: () => reconcileCloudAgents({ prisma, jobs, cloudAgent }), reconcileComputerUpdates: () => reconcileComputerUpdates({ prisma, jobs }), reconcileProfileConsolidation: createProfileConsolidationScheduler({ prisma, jobs }), reconcileDelegatedTasks: () => reconcileDelegatedTasks({ prisma, jobs, events }), }); reconciler.start(); let stopping = false; const stop = async () => { if (stopping) return; stopping = true; try { await reconciler.stop(); mediaAbort.abort(); await jobHost.stop(); await jobs.close(); await realtime.close(); await connector.stop(); await mcp.close(); await prisma.$disconnect().catch(() => undefined); await pool.end().catch(() => undefined); } finally { await logger.flush({ timeoutMs: 2_000 }); } }; process.once("SIGTERM", () => void stop()); process.once("SIGINT", () => void stop()); // graphile-worker fires completeJob() without awaiting it. When pool.connect() // then hits Postgres 53300, that rejection is unhandled. Exiting here is the // crash loop: Docker restarts the process before Postgres has reaped the old // backends, so the next boot cannot connect either. Stay up on that rejection // only — do not resume after uncaughtException (Node leaves the process in an // undefined state). process.on("uncaughtException", (error) => { logger.error("uncaughtException", error); void stop().finally(() => process.exit(1)); }); process.on("unhandledRejection", (reason) => { logger.error("unhandledRejection", reason); if (isTooManyDatabaseConnections(reason)) return; void stop().finally(() => process.exit(1)); }); logger.info("worker ready"); } main().catch(async (error) => { logger.error("worker startup failed", error); await logger.flush({ timeoutMs: 2_000 }); // jobHost.start retries 53300 without bound above, so a saturated Postgres at // that step does not reach here. Other startup failures still exit. process.exit(1); }); // reload