6c7eed7580
Fresh history cut from cynarAI/evoo antigravity-handoff/project-timeline (Voice First + project timeline). Brand docs/titles as evOo; keep @rakazo/* packages as tech debt. Excludes node_modules, caches, secrets, voice blobs.
333 lines
11 KiB
TypeScript
333 lines
11 KiB
TypeScript
import {
|
|
type BackgroundJob,
|
|
type BackgroundJobHandlers,
|
|
dispatchBackgroundJob,
|
|
type JobPublisher,
|
|
type JobWorkerHost,
|
|
} from "@rakazo/adapter-kit";
|
|
import { isTooManyDatabaseConnections } from "@rakazo/db";
|
|
import { runCorrelatedJob, unwrapJobPayload, wrapJobPayload } from "@rakazo/logging";
|
|
import { makeWorkerUtils, type Runner, run, type WorkerUtils } from "graphile-worker";
|
|
import type { Pool } from "pg";
|
|
|
|
// Share the caller's pg.Pool instead of opening a separate connectionString-based
|
|
// pool per graphile-worker component: three independent pools per worker process
|
|
// (Prisma + publisher + runner) triples the connection footprint against Postgres'
|
|
// max_connections for no benefit, since they never need isolation from each other.
|
|
export class GraphileJobPublisher implements JobPublisher {
|
|
private utils: Promise<WorkerUtils> | undefined;
|
|
private closed = false;
|
|
|
|
constructor(private readonly pgPool: Pool) {}
|
|
|
|
async enqueue(job: BackgroundJob): Promise<void> {
|
|
const utils = await this.getUtils();
|
|
await utils.addJob(job.name, wrapJobPayload(job.payload), {
|
|
runAt: job.availableAt,
|
|
jobKey: job.replaceKey,
|
|
});
|
|
}
|
|
|
|
async cancel(key: string): Promise<void> {
|
|
const utils = await this.getUtils();
|
|
await utils.withPgClient(async (client) => {
|
|
await client.query("select graphile_worker.remove_job($1::text)", [key]);
|
|
});
|
|
}
|
|
|
|
async close(): Promise<void> {
|
|
if (this.closed) return;
|
|
this.closed = true;
|
|
if (this.utils) await (await this.utils).release();
|
|
}
|
|
|
|
private getUtils(): Promise<WorkerUtils> {
|
|
if (this.closed) throw new Error("Background job publisher is closed");
|
|
this.utils ??= makeWorkerUtils({ pgPool: this.pgPool });
|
|
return this.utils;
|
|
}
|
|
}
|
|
|
|
/** Same bounded backoff the worker uses around jobHost.start for 53300. */
|
|
export function databaseCapacityBackoffMs(attempt: number): number {
|
|
return Math.min(30_000, 200 * 2 ** Math.min(attempt, 8));
|
|
}
|
|
|
|
export class GraphileJobWorkerHost implements JobWorkerHost {
|
|
private runner: Runner | undefined;
|
|
private handlers: BackgroundJobHandlers | undefined;
|
|
private stopping = false;
|
|
private superviseTask: Promise<void> | undefined;
|
|
private wakeSleep: (() => void) | undefined;
|
|
|
|
constructor(
|
|
private readonly pgPool: Pool,
|
|
private readonly options: {
|
|
concurrency?: number;
|
|
pollInterval?: number;
|
|
noHandleSignals?: boolean;
|
|
/** Test seam: delay between 53300 restart attempts. */
|
|
sleep?: (ms: number) => Promise<void>;
|
|
} = {},
|
|
) {}
|
|
|
|
async start(handlers: BackgroundJobHandlers): Promise<void> {
|
|
if (this.runner || this.superviseTask) return;
|
|
this.stopping = false;
|
|
this.handlers = handlers;
|
|
await this.launchRunner();
|
|
// run() resolves once the runner is up; runner.promise can still reject later
|
|
// (e.g. Postgres 53300). Observe it so a dead runner cannot leave the worker
|
|
// process idle forever while unhandledRejection swallows that same error.
|
|
this.superviseTask = this.supervise();
|
|
}
|
|
|
|
async stop(): Promise<void> {
|
|
this.stopping = true;
|
|
this.wakeSleep?.();
|
|
try {
|
|
await this.runner?.stop();
|
|
} finally {
|
|
await this.superviseTask?.catch(() => undefined);
|
|
this.runner = undefined;
|
|
this.superviseTask = undefined;
|
|
this.handlers = undefined;
|
|
}
|
|
}
|
|
|
|
private async launchRunner(): Promise<void> {
|
|
const handlers = this.handlers;
|
|
if (!handlers) throw new Error("Background job worker has no handlers");
|
|
const taskList = Object.fromEntries(
|
|
Object.keys(handlers).map((name) => [
|
|
name,
|
|
async (payload: unknown) => {
|
|
const unpacked = unwrapJobPayload(payload);
|
|
await runCorrelatedJob({
|
|
name,
|
|
payload: unpacked.payload,
|
|
correlation: unpacked.correlation,
|
|
run: () => dispatchBackgroundJob(handlers, name, unpacked.payload),
|
|
});
|
|
},
|
|
]),
|
|
);
|
|
const runner = await run({
|
|
pgPool: this.pgPool,
|
|
concurrency: this.options.concurrency ?? 4,
|
|
pollInterval: this.options.pollInterval ?? 500,
|
|
noHandleSignals: this.options.noHandleSignals,
|
|
taskList,
|
|
});
|
|
if (this.stopping) {
|
|
await runner.stop().catch(() => undefined);
|
|
return;
|
|
}
|
|
this.runner = runner;
|
|
}
|
|
|
|
private delay(ms: number): Promise<void> {
|
|
if (this.stopping) return Promise.resolve();
|
|
const sleep =
|
|
this.options.sleep ??
|
|
((wait: number) => new Promise<void>((resolve) => setTimeout(resolve, wait)));
|
|
return new Promise((resolve) => {
|
|
let settled = false;
|
|
const finish = () => {
|
|
if (settled) return;
|
|
settled = true;
|
|
this.wakeSleep = undefined;
|
|
resolve();
|
|
};
|
|
this.wakeSleep = finish;
|
|
void Promise.resolve(sleep(ms)).then(finish, finish);
|
|
});
|
|
}
|
|
|
|
private async supervise(): Promise<void> {
|
|
for (;;) {
|
|
const runner = this.runner;
|
|
if (!runner || this.stopping) return;
|
|
try {
|
|
await runner.promise;
|
|
return;
|
|
} catch (error) {
|
|
if (this.stopping) return;
|
|
if (this.runner === runner) this.runner = undefined;
|
|
if (!isTooManyDatabaseConnections(error)) throw error;
|
|
// Back off after every lifecycle 53300 (runner death or failed relaunch),
|
|
// not only when launchRunner rejects — otherwise a runner that starts then
|
|
// dies again under saturation reconnects with no delay.
|
|
for (let attempt = 0; ; attempt += 1) {
|
|
if (this.stopping) return;
|
|
await this.delay(databaseCapacityBackoffMs(attempt));
|
|
if (this.stopping) return;
|
|
try {
|
|
await this.launchRunner();
|
|
if (this.stopping || !this.runner) return;
|
|
break;
|
|
} catch (startError) {
|
|
if (!isTooManyDatabaseConnections(startError)) throw startError;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
interface QueuedJob {
|
|
name: BackgroundJob["name"];
|
|
payload: unknown;
|
|
availableAt?: Date;
|
|
replaceKey?: string;
|
|
}
|
|
|
|
function toQueuedJob(job: BackgroundJob): QueuedJob {
|
|
return {
|
|
name: job.name,
|
|
payload: wrapJobPayload(job.payload),
|
|
availableAt: job.availableAt,
|
|
replaceKey: job.replaceKey,
|
|
};
|
|
}
|
|
|
|
export class InMemoryJobQueue implements JobPublisher, JobWorkerHost {
|
|
private handlers: BackgroundJobHandlers | undefined;
|
|
private readonly timers = new Set<ReturnType<typeof setTimeout>>();
|
|
private readonly scheduled = new Map<ReturnType<typeof setTimeout>, QueuedJob>();
|
|
private readonly keyed = new Map<string, ReturnType<typeof setTimeout>>();
|
|
private readonly active = new Set<Promise<void>>();
|
|
private readonly closingJobs: QueuedJob[] = [];
|
|
private draining: Promise<void> | undefined;
|
|
private closed = false;
|
|
private stopped = false;
|
|
private closing = false;
|
|
private acceptingClosingJobs = false;
|
|
private closeRequested = false;
|
|
|
|
async enqueue(job: BackgroundJob): Promise<void> {
|
|
const stored = toQueuedJob(job);
|
|
if (this.closed) throw new Error("Background job publisher is closed");
|
|
if (this.stopped) throw new Error("Background job publisher is stopped");
|
|
if (this.closing) {
|
|
this.enqueueWhileClosing(stored);
|
|
return;
|
|
}
|
|
if (stored.replaceKey) {
|
|
await this.cancel(stored.replaceKey);
|
|
if (this.closed) throw new Error("Background job publisher is closed");
|
|
if (this.stopped) throw new Error("Background job publisher is stopped");
|
|
if (this.closing) {
|
|
this.enqueueWhileClosing(stored);
|
|
return;
|
|
}
|
|
}
|
|
const delay = stored.availableAt ? Math.max(0, stored.availableAt.getTime() - Date.now()) : 0;
|
|
const timer = setTimeout(() => {
|
|
this.timers.delete(timer);
|
|
this.scheduled.delete(timer);
|
|
if (stored.replaceKey && this.keyed.get(stored.replaceKey) === timer) {
|
|
this.keyed.delete(stored.replaceKey);
|
|
}
|
|
const handlers = this.handlers;
|
|
if (!handlers) return;
|
|
void this.dispatch(handlers, stored);
|
|
}, delay);
|
|
this.timers.add(timer);
|
|
this.scheduled.set(timer, stored);
|
|
if (stored.replaceKey) this.keyed.set(stored.replaceKey, timer);
|
|
}
|
|
|
|
async cancel(replaceKey: string): Promise<void> {
|
|
const closing = this.closingJobs.findIndex((job) => job.replaceKey === replaceKey);
|
|
if (closing >= 0) this.closingJobs.splice(closing, 1);
|
|
const timer = this.keyed.get(replaceKey);
|
|
if (!timer) return;
|
|
clearTimeout(timer);
|
|
this.keyed.delete(replaceKey);
|
|
this.timers.delete(timer);
|
|
this.scheduled.delete(timer);
|
|
}
|
|
|
|
async start(handlers: BackgroundJobHandlers): Promise<void> {
|
|
this.handlers = handlers;
|
|
this.stopped = false;
|
|
}
|
|
|
|
async stop(): Promise<void> {
|
|
await this.drain();
|
|
}
|
|
|
|
private dispatch(handlers: BackgroundJobHandlers, job: QueuedJob): Promise<void> {
|
|
const unpacked = unwrapJobPayload(job.payload);
|
|
const active = runCorrelatedJob({
|
|
name: job.name,
|
|
payload: unpacked.payload,
|
|
correlation: unpacked.correlation,
|
|
run: () => dispatchBackgroundJob(handlers, job.name, unpacked.payload),
|
|
}).catch(() => undefined);
|
|
this.active.add(active);
|
|
void active.finally(() => this.active.delete(active));
|
|
return active;
|
|
}
|
|
|
|
async close(): Promise<void> {
|
|
if (this.closed) return;
|
|
this.closeRequested = true;
|
|
await this.drain();
|
|
}
|
|
|
|
private enqueueWhileClosing(job: QueuedJob): void {
|
|
if (job.replaceKey) {
|
|
const existing = this.closingJobs.findIndex((queued) => queued.replaceKey === job.replaceKey);
|
|
if (existing >= 0) this.closingJobs.splice(existing, 1);
|
|
}
|
|
if (job.availableAt && job.availableAt.getTime() > Date.now()) {
|
|
// Delayed in-memory jobs are intentionally discarded on shutdown; durable
|
|
// schedulers reconcile them after restart. Never run them early.
|
|
return;
|
|
}
|
|
if (!this.acceptingClosingJobs) throw new Error("Background job publisher is closing");
|
|
this.closingJobs.push(job);
|
|
}
|
|
|
|
private async drain(): Promise<void> {
|
|
if (this.draining) return this.draining;
|
|
const draining = this.performDrain();
|
|
this.draining = draining;
|
|
try {
|
|
await draining;
|
|
} finally {
|
|
if (this.draining === draining) this.draining = undefined;
|
|
}
|
|
}
|
|
|
|
private async performDrain(): Promise<void> {
|
|
this.closing = true;
|
|
this.acceptingClosingJobs = true;
|
|
for (const timer of this.timers) {
|
|
clearTimeout(timer);
|
|
const job = this.scheduled.get(timer);
|
|
if (job) this.enqueueWhileClosing(job);
|
|
}
|
|
this.timers.clear();
|
|
this.scheduled.clear();
|
|
this.keyed.clear();
|
|
await Promise.all(this.active);
|
|
this.acceptingClosingJobs = false;
|
|
const handlers = this.handlers;
|
|
const closingJobs = this.closingJobs.splice(0);
|
|
if (!handlers && closingJobs.length > 0) {
|
|
throw new Error("Background job publisher is closing");
|
|
}
|
|
if (handlers) {
|
|
for (const job of closingJobs) void this.dispatch(handlers, job);
|
|
}
|
|
await Promise.all(this.active);
|
|
this.handlers = undefined;
|
|
this.closed = this.closeRequested;
|
|
this.stopped = !this.closeRequested;
|
|
this.closing = false;
|
|
}
|
|
}
|