import type { BackgroundJobHandlers } from "@rakazo/adapter-kit"; import type { Runner } from "graphile-worker"; import type { Pool } from "pg"; import { afterEach, describe, expect, it, vi } from "vitest"; const run = vi.hoisted(() => vi.fn()); vi.mock("graphile-worker", () => ({ run: (...args: unknown[]) => run(...args), makeWorkerUtils: vi.fn(), })); import { databaseCapacityBackoffMs, GraphileJobWorkerHost } from "./wakeup.js"; function handlers(): BackgroundJobHandlers { return { "run.continue": vi.fn(async () => undefined), "routine.wakeup": vi.fn(async () => undefined), "computer.update": vi.fn(async () => undefined), "computer.sleep": vi.fn(async () => undefined), "computer.control-expire": vi.fn(async () => undefined), "skill.teaching-expire": vi.fn(async () => undefined), "history.compact": vi.fn(async () => undefined), "messaging.deliver": vi.fn(async () => undefined), "cloud_agent.poll": vi.fn(async () => undefined), "profile.consolidate": vi.fn(async () => undefined), "media.run": vi.fn(async () => undefined), "media.library.sync": vi.fn(async () => undefined), }; } function deferred() { let resolve!: () => void; let reject!: (reason?: unknown) => void; const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); return { promise, resolve, reject }; } function mockRunner() { const life = deferred(); // Keep a rejection handler so Node does not treat the mock lifecycle as an // unexpected unhandledRejection before the host attaches supervise(). life.promise.catch(() => undefined); const runner = { promise: life.promise, stop: vi.fn(async () => { life.resolve(); }), kill: vi.fn(async () => undefined), addJob: vi.fn(), events: { on: vi.fn(), off: vi.fn(), once: vi.fn(), emit: vi.fn() }, } as unknown as Runner; return { runner, rejectLife: (error: unknown) => life.reject(error), }; } const tooMany = Object.assign(new Error("sorry, too many clients already"), { code: "53300" }); describe("databaseCapacityBackoffMs", () => { it("matches the worker startup backoff curve", () => { expect(databaseCapacityBackoffMs(0)).toBe(200); expect(databaseCapacityBackoffMs(3)).toBe(1_600); expect(databaseCapacityBackoffMs(8)).toBe(30_000); expect(databaseCapacityBackoffMs(20)).toBe(30_000); }); }); describe("GraphileJobWorkerHost runner lifecycle", () => { afterEach(() => { run.mockReset(); }); it("backs off then restarts when runner.promise rejects with 53300", async () => { const first = mockRunner(); const second = mockRunner(); run.mockResolvedValueOnce(first.runner).mockResolvedValueOnce(second.runner); const sleeps: number[] = []; const host = new GraphileJobWorkerHost({} as Pool, { sleep: async (ms) => { sleeps.push(ms); }, }); await host.start(handlers()); expect(run).toHaveBeenCalledTimes(1); first.rejectLife(tooMany); await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(2)); expect(sleeps).toEqual([200]); await host.stop(); expect(second.runner.stop).toHaveBeenCalled(); }); it("retries launch with escalating backoff when restart start hits 53300", async () => { const first = mockRunner(); const recovered = mockRunner(); run .mockResolvedValueOnce(first.runner) .mockRejectedValueOnce(tooMany) .mockResolvedValueOnce(recovered.runner); const sleeps: number[] = []; const host = new GraphileJobWorkerHost({} as Pool, { sleep: async (ms) => { sleeps.push(ms); }, }); await host.start(handlers()); first.rejectLife(tooMany); await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(3)); expect(sleeps).toEqual([200, 400]); await host.stop(); expect(recovered.runner.stop).toHaveBeenCalled(); }); it("does not restart when runner.promise rejects for a non-53300 error", async () => { const first = mockRunner(); run.mockResolvedValueOnce(first.runner); const host = new GraphileJobWorkerHost({} as Pool, { sleep: async () => undefined, }); await host.start(handlers()); const superviseTask = (host as unknown as { superviseTask: Promise }).superviseTask; first.rejectLife(new Error("runner exploded")); await expect(superviseTask).rejects.toThrow("runner exploded"); expect(run).toHaveBeenCalledTimes(1); await host.stop(); }); it("wakes a pending restart delay when stop is called", async () => { const first = mockRunner(); run.mockResolvedValueOnce(first.runner); const sleepStarted = deferred(); const host = new GraphileJobWorkerHost({} as Pool, { sleep: () => { sleepStarted.resolve(); return new Promise(() => undefined); }, }); await host.start(handlers()); first.rejectLife(tooMany); await sleepStarted.promise; await host.stop(); expect(run).toHaveBeenCalledTimes(1); }); });