import { type BackgroundJob, type BackgroundJobHandlers, dispatchBackgroundJob, type JobPublisher, type JobWorkerHost, } from "@rakazo/adapter-kit"; import { isTooManyDatabaseConnections } from "@rakazo/db"; import { runCorrelatedJob, unwrapJobPayload, wrapJobPayload } from "@rakazo/logging"; import { makeWorkerUtils, type Runner, run, type WorkerUtils } from "graphile-worker"; import type { Pool } from "pg"; // Share the caller's pg.Pool instead of opening a separate connectionString-based // pool per graphile-worker component: three independent pools per worker process // (Prisma + publisher + runner) triples the connection footprint against Postgres' // max_connections for no benefit, since they never need isolation from each other. export class GraphileJobPublisher implements JobPublisher { private utils: Promise | undefined; private closed = false; constructor(private readonly pgPool: Pool) {} async enqueue(job: BackgroundJob): Promise { const utils = await this.getUtils(); await utils.addJob(job.name, wrapJobPayload(job.payload), { runAt: job.availableAt, jobKey: job.replaceKey, queueName: job.queue, }); } async cancel(key: string): Promise { const utils = await this.getUtils(); await utils.withPgClient(async (client) => { await client.query("select graphile_worker.remove_job($1::text)", [key]); }); } async close(): Promise { if (this.closed) return; this.closed = true; if (this.utils) await (await this.utils).release(); } private getUtils(): Promise { if (this.closed) throw new Error("Background job publisher is closed"); this.utils ??= makeWorkerUtils({ pgPool: this.pgPool }); return this.utils; } } /** Same bounded backoff the worker uses around jobHost.start for 53300. */ export function databaseCapacityBackoffMs(attempt: number): number { return Math.min(30_000, 200 * 2 ** Math.min(attempt, 8)); } export class GraphileJobWorkerHost implements JobWorkerHost { private runner: Runner | undefined; private handlers: BackgroundJobHandlers | undefined; private stopping = false; private superviseTask: Promise | undefined; private wakeSleep: (() => void) | undefined; constructor( private readonly pgPool: Pool, private readonly options: { concurrency?: number; pollInterval?: number; noHandleSignals?: boolean; /** Test seam: delay between 53300 restart attempts. */ sleep?: (ms: number) => Promise; } = {}, ) {} async start(handlers: BackgroundJobHandlers): Promise { if (this.runner || this.superviseTask) return; this.stopping = false; this.handlers = handlers; await this.launchRunner(); // run() resolves once the runner is up; runner.promise can still reject later // (e.g. Postgres 53300). Observe it so a dead runner cannot leave the worker // process idle forever while unhandledRejection swallows that same error. this.superviseTask = this.supervise(); } async stop(): Promise { this.stopping = true; this.wakeSleep?.(); try { await this.runner?.stop(); } finally { await this.superviseTask?.catch(() => undefined); this.runner = undefined; this.superviseTask = undefined; this.handlers = undefined; } } private async launchRunner(): Promise { const handlers = this.handlers; if (!handlers) throw new Error("Background job worker has no handlers"); const taskList = Object.fromEntries( Object.keys(handlers).map((name) => [ name, async (payload: unknown) => { const unpacked = unwrapJobPayload(payload); await runCorrelatedJob({ name, payload: unpacked.payload, correlation: unpacked.correlation, run: () => dispatchBackgroundJob(handlers, name, unpacked.payload), }); }, ]), ); const runner = await run({ pgPool: this.pgPool, concurrency: this.options.concurrency ?? 4, pollInterval: this.options.pollInterval ?? 500, noHandleSignals: this.options.noHandleSignals, taskList, }); if (this.stopping) { await runner.stop().catch(() => undefined); return; } this.runner = runner; } private delay(ms: number): Promise { if (this.stopping) return Promise.resolve(); const sleep = this.options.sleep ?? ((wait: number) => new Promise((resolve) => setTimeout(resolve, wait))); return new Promise((resolve) => { let settled = false; const finish = () => { if (settled) return; settled = true; this.wakeSleep = undefined; resolve(); }; this.wakeSleep = finish; void Promise.resolve(sleep(ms)).then(finish, finish); }); } private async supervise(): Promise { for (;;) { const runner = this.runner; if (!runner || this.stopping) return; try { await runner.promise; return; } catch (error) { if (this.stopping) return; if (this.runner === runner) this.runner = undefined; if (!isTooManyDatabaseConnections(error)) throw error; // Back off after every lifecycle 53300 (runner death or failed relaunch), // not only when launchRunner rejects — otherwise a runner that starts then // dies again under saturation reconnects with no delay. for (let attempt = 0; ; attempt += 1) { if (this.stopping) return; await this.delay(databaseCapacityBackoffMs(attempt)); if (this.stopping) return; try { await this.launchRunner(); if (this.stopping || !this.runner) return; break; } catch (startError) { if (!isTooManyDatabaseConnections(startError)) throw startError; } } } } } } interface QueuedJob { name: BackgroundJob["name"]; payload: unknown; availableAt?: Date; replaceKey?: string; queue?: string; } function toQueuedJob(job: BackgroundJob): QueuedJob { return { name: job.name, payload: wrapJobPayload(job.payload), availableAt: job.availableAt, replaceKey: job.replaceKey, queue: job.queue, }; } export class InMemoryJobQueue implements JobPublisher, JobWorkerHost { private handlers: BackgroundJobHandlers | undefined; private readonly timers = new Set>(); private readonly scheduled = new Map, QueuedJob>(); private readonly keyed = new Map>(); private readonly active = new Set>(); /** Last job of each named queue; the next one starts after it settles, like Graphile queues. */ private readonly queueTails = new Map>(); private readonly closingJobs: QueuedJob[] = []; private draining: Promise | undefined; private closed = false; private stopped = false; private closing = false; private acceptingClosingJobs = false; private closeRequested = false; async enqueue(job: BackgroundJob): Promise { const stored = toQueuedJob(job); if (this.closed) throw new Error("Background job publisher is closed"); if (this.stopped) throw new Error("Background job publisher is stopped"); if (this.closing) { this.enqueueWhileClosing(stored); return; } if (stored.replaceKey) { await this.cancel(stored.replaceKey); if (this.closed) throw new Error("Background job publisher is closed"); if (this.stopped) throw new Error("Background job publisher is stopped"); if (this.closing) { this.enqueueWhileClosing(stored); return; } } const delay = stored.availableAt ? Math.max(0, stored.availableAt.getTime() - Date.now()) : 0; const timer = setTimeout(() => { this.timers.delete(timer); this.scheduled.delete(timer); if (stored.replaceKey && this.keyed.get(stored.replaceKey) === timer) { this.keyed.delete(stored.replaceKey); } const handlers = this.handlers; if (!handlers) return; void this.dispatch(handlers, stored); }, delay); this.timers.add(timer); this.scheduled.set(timer, stored); if (stored.replaceKey) this.keyed.set(stored.replaceKey, timer); } async cancel(replaceKey: string): Promise { const closing = this.closingJobs.findIndex((job) => job.replaceKey === replaceKey); if (closing >= 0) this.closingJobs.splice(closing, 1); const timer = this.keyed.get(replaceKey); if (!timer) return; clearTimeout(timer); this.keyed.delete(replaceKey); this.timers.delete(timer); this.scheduled.delete(timer); } async start(handlers: BackgroundJobHandlers): Promise { this.handlers = handlers; this.stopped = false; } async stop(): Promise { await this.drain(); } private dispatch(handlers: BackgroundJobHandlers, job: QueuedJob): Promise { const unpacked = unwrapJobPayload(job.payload); const run = () => runCorrelatedJob({ name: job.name, payload: unpacked.payload, correlation: unpacked.correlation, run: () => dispatchBackgroundJob(handlers, job.name, unpacked.payload), }); const queue = job.queue; const tail = queue ? this.queueTails.get(queue) : undefined; const active = (tail ? tail.then(run) : run()).catch(() => undefined); if (queue) { this.queueTails.set(queue, active); void active.finally(() => { if (this.queueTails.get(queue) === active) this.queueTails.delete(queue); }); } this.active.add(active); void active.finally(() => this.active.delete(active)); return active; } async close(): Promise { if (this.closed) return; this.closeRequested = true; await this.drain(); } private enqueueWhileClosing(job: QueuedJob): void { if (job.replaceKey) { const existing = this.closingJobs.findIndex((queued) => queued.replaceKey === job.replaceKey); if (existing >= 0) this.closingJobs.splice(existing, 1); } if (job.availableAt && job.availableAt.getTime() > Date.now()) { // Delayed in-memory jobs are intentionally discarded on shutdown; durable // schedulers reconcile them after restart. Never run them early. return; } if (!this.acceptingClosingJobs) throw new Error("Background job publisher is closing"); this.closingJobs.push(job); } private async drain(): Promise { if (this.draining) return this.draining; const draining = this.performDrain(); this.draining = draining; try { await draining; } finally { if (this.draining === draining) this.draining = undefined; } } private async performDrain(): Promise { this.closing = true; this.acceptingClosingJobs = true; for (const timer of this.timers) { clearTimeout(timer); const job = this.scheduled.get(timer); if (job) this.enqueueWhileClosing(job); } this.timers.clear(); this.scheduled.clear(); this.keyed.clear(); await Promise.all(this.active); this.acceptingClosingJobs = false; const handlers = this.handlers; const closingJobs = this.closingJobs.splice(0); if (!handlers && closingJobs.length > 0) { throw new Error("Background job publisher is closing"); } if (handlers) { for (const job of closingJobs) void this.dispatch(handlers, job); } await Promise.all(this.active); this.handlers = undefined; this.closed = this.closeRequested; this.stopped = !this.closeRequested; this.closing = false; } }