import type * as NodeFsPromises from "node:fs/promises"; import http from "node:http"; import path from "node:path"; import { Readable } from "node:stream"; import { resolveSupervisorToken } from "@rakazo/core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { COMPUTER_IMAGE, computerNetworkNameFor, hostComputerUser } from "./computer-spec.js"; const mocks = vi.hoisted(() => ({ docker: { version: vi.fn(), getImage: vi.fn(), getContainer: vi.fn(), listContainers: vi.fn(), createContainer: vi.fn(), createNetwork: vi.fn(), }, assertHomeWritable: vi.fn(), })); vi.mock("dockerode", () => ({ default: class { version = mocks.docker.version; getImage = mocks.docker.getImage; getContainer = mocks.docker.getContainer; listContainers = mocks.docker.listContainers; createContainer = mocks.docker.createContainer; createNetwork = mocks.docker.createNetwork; }, })); vi.mock("./home-ownership.js", () => ({ assertComputerHomeWritable: mocks.assertHomeWritable })); vi.mock("node:fs/promises", async (importOriginal) => ({ ...(await importOriginal()), mkdir: vi.fn(), })); let screen: http.Server; let screenPort: string; beforeEach(async () => { vi.resetModules(); vi.resetAllMocks(); vi.stubEnv("HOSTNAME", ""); vi.stubEnv("DATA_DIR", "/tmp/rakazo-loopback-test"); vi.stubEnv("SANDBOX_SCREEN_NETWORK", "published"); vi.stubEnv("SANDBOX_SCREEN_HOST", "127.0.0.1"); screen = http.createServer((_req, res) => res.end("ok")); await new Promise((resolve) => screen.listen(0, "127.0.0.1", resolve)); const address = screen.address(); if (!address || typeof address === "string") throw new Error("expected a TCP address"); screenPort = String(address.port); }); afterEach(async () => { vi.unstubAllEnvs(); await new Promise((resolve) => { screen.close(() => resolve()); screen.closeAllConnections(); }); }); describe("computer loopback provision lifecycle", () => { it.each([ { error: new Error("daemon unavailable"), status: 500 }, { error: Object.assign(new Error("permission denied"), { statusCode: 403 }), status: 500 }, { error: Object.assign(new Error("container missing"), { statusCode: 404 }), status: 404 }, ])("reports inspection failures correctly when stopping: $status", async ({ error, status }) => { const { supervisorApp } = await import("./index.js"); const container = { inspect: vi.fn().mockRejectedValue(error), stop: vi.fn(), exec: vi.fn() }; mocks.docker.getContainer.mockReturnValue(container); const response = await supervisorApp.request("/computers/inspect-failure/stop", { method: "POST", headers: { authorization: `Bearer ${resolveSupervisorToken(process.env)}`, "x-rakazo-bot-id": "bot", "x-rakazo-space-id": "space", }, }); expect(response.status).toBe(status); expect(container.stop).not.toHaveBeenCalled(); expect(container.exec).not.toHaveBeenCalled(); }); it("rejects another computer identity without stopping its container", async () => { const { supervisorApp } = await import("./index.js"); const container = { inspect: vi.fn().mockResolvedValue({ Config: { Labels: { "rakazo.managed": "true", "rakazo.botId": "other", "rakazo.spaceId": "other" }, }, }), stop: vi.fn(), exec: vi.fn(), }; mocks.docker.getContainer.mockReturnValue(container); const response = await supervisorApp.request("/computers/identity-mismatch/stop", { method: "POST", headers: { authorization: `Bearer ${resolveSupervisorToken(process.env)}`, "x-rakazo-bot-id": "bot", "x-rakazo-space-id": "space", }, }); expect(response.status).toBe(403); expect(container.stop).not.toHaveBeenCalled(); expect(container.exec).not.toHaveBeenCalled(); }); it("rechecks stopped state after a concurrent stop owns the screen lock", async () => { const { supervisorApp } = await import("./index.js"); let running = true; let releaseStop!: () => void; let stopStarted!: () => void; const started = new Promise((resolve) => { stopStarted = resolve; }); const stopped = new Promise((resolve) => { releaseStop = resolve; }); const container = { inspect: vi.fn(async () => ({ Config: { Labels: { "rakazo.managed": "true", "rakazo.botId": "bot", "rakazo.spaceId": "space" }, }, State: { Running: running }, })), exec: vi.fn(async () => { if (!running) throw new Error("container stopped"); return { start: async () => Readable.from([]), inspect: async () => ({ ExitCode: 0 }) }; }), stop: vi.fn(async () => { stopStarted(); await stopped; running = false; }), }; mocks.docker.getContainer.mockReturnValue(container); const stop = () => supervisorApp.request("/computers/concurrent-stop/stop", { method: "POST", headers: { authorization: `Bearer ${resolveSupervisorToken(process.env)}`, "x-rakazo-bot-id": "bot", "x-rakazo-space-id": "space", }, }); const first = stop(); await started; const second = stop(); await vi.waitFor(() => expect(container.inspect).toHaveBeenCalledTimes(3)); releaseStop(); expect((await first).status).toBe(200); expect((await second).status).toBe(200); expect(container.stop).toHaveBeenCalledOnce(); expect(container.exec).toHaveBeenCalledOnce(); }); it("stops the computer after a failed checkpoint while reporting the failure", async () => { const { supervisorApp } = await import("./index.js"); const container = { inspect: vi.fn(async () => ({ Config: { Labels: { "rakazo.managed": "true", "rakazo.botId": "bot", "rakazo.spaceId": "space" }, }, State: { Running: true }, })), exec: vi.fn(async () => ({ start: async () => Readable.from([]), inspect: async () => ({ ExitCode: 1 }), })), stop: vi.fn(async () => {}), }; mocks.docker.getContainer.mockReturnValue(container); const response = await supervisorApp.request("/computers/failed-checkpoint/stop", { method: "POST", headers: { authorization: `Bearer ${resolveSupervisorToken(process.env)}`, "x-rakazo-bot-id": "bot", "x-rakazo-space-id": "space", }, }); expect(response.status).toBe(500); expect(container.stop).toHaveBeenCalledOnce(); }); it.each([ { enabled: true, hosts: [], resumed: false }, { enabled: true, hosts: ["127.0.0.1"], resumed: true }, { enabled: true, hosts: ["0.0.0.0", "127.0.0.1"], resumed: false }, { enabled: false, hosts: ["127.0.0.1"], resumed: false }, { enabled: false, hosts: ["0.0.0.0"], resumed: false }, { enabled: false, hosts: [], resumed: true }, ])("matches publication on stopped container reuse: %j", async ({ enabled, hosts, resumed }) => { vi.stubEnv("SANDBOX_CONTROL_VIA_LOOPBACK", String(enabled)); const { supervisorApp } = await import("./index.js"); const homePath = path.join(process.env.DATA_DIR!, "homes", "bot"); const info = { Image: "test-image-id", Config: { User: hostComputerUser(), Labels: { "rakazo.managed": "true", "rakazo.botId": "bot", "rakazo.spaceId": "space" }, }, HostConfig: { NetworkMode: computerNetworkNameFor("bot"), PortBindings: { "7070/tcp": hosts.map((HostIp) => ({ HostIp, HostPort: "0" })) }, }, State: { Running: false }, NetworkSettings: { Ports: { "6080/tcp": [{ HostIp: "127.0.0.1", HostPort: screenPort }] } }, }; const existing = { id: "existing", inspect: vi.fn().mockResolvedValue(info), start: vi.fn().mockResolvedValue(undefined), remove: vi.fn().mockResolvedValue(undefined), }; const replacement = { id: "replacement", inspect: vi.fn().mockResolvedValue(info), start: vi.fn().mockResolvedValue(undefined), }; mocks.docker.getImage.mockReturnValue({ inspect: vi.fn().mockResolvedValue({ Id: info.Image }), }); mocks.docker.getContainer.mockReturnValue(existing); mocks.docker.listContainers.mockResolvedValue([{ Id: existing.id }]); mocks.docker.createContainer.mockResolvedValue(replacement); mocks.docker.createNetwork.mockResolvedValue({}); const response = await supervisorApp.request("/computers", { method: "POST", headers: { authorization: `Bearer ${resolveSupervisorToken(process.env)}`, "content-type": "application/json", "x-rakazo-bot-id": "bot", "x-rakazo-space-id": "space", }, body: JSON.stringify({ botId: "bot", spaceId: "space", homePath }), }); expect(await response.json()).toMatchObject({ resumed, id: resumed ? "existing" : "replacement", }); expect(response.status).toBe(200); if (resumed) { expect(mocks.docker.createNetwork).not.toHaveBeenCalled(); expect(existing.start).toHaveBeenCalledOnce(); expect(existing.remove).not.toHaveBeenCalled(); expect(mocks.docker.createContainer).not.toHaveBeenCalled(); } else { expect(existing.remove).toHaveBeenCalledWith({ force: true }); expect(replacement.start).toHaveBeenCalledOnce(); const [options] = mocks.docker.createContainer.mock.calls[0]!; expect(options.HostConfig.PortBindings["7070/tcp"]).toEqual( enabled ? [{ HostIp: "127.0.0.1", HostPort: "0" }] : undefined, ); expect(options.HostConfig.Binds).toEqual([`${homePath}:/home/rakazo`]); expect(options.Env).toContainEqual( expect.stringMatching(/^RAKAZO_COMPUTER_CONTROL_TOKEN=.+/), ); } }); }); describe("provisioning network rollback", () => { function fixture() { const network = { remove: vi.fn().mockResolvedValue(undefined) }; const container = { id: "new-computer", start: vi.fn().mockResolvedValue(undefined), remove: vi.fn().mockResolvedValue(undefined), }; mocks.docker.getImage.mockReturnValue({ inspect: vi.fn().mockResolvedValue({ Id: "image" }) }); mocks.docker.listContainers.mockResolvedValue([]); mocks.docker.createNetwork.mockResolvedValue(network); mocks.docker.createContainer.mockResolvedValue(container); return { network, container }; } async function provision(homePath = path.join(process.env.DATA_DIR!, "homes", "bot")) { const { supervisorApp } = await import("./index.js"); return supervisorApp.request("/computers", { method: "POST", headers: { authorization: `Bearer ${resolveSupervisorToken(process.env)}`, "content-type": "application/json", "x-rakazo-bot-id": "bot", "x-rakazo-space-id": "space", }, body: JSON.stringify({ botId: "bot", spaceId: "space", homePath }), }); } it.each(["1.44", "1.45"])( "provisions named-volume homes only with subpath support (%s)", async (apiVersion) => { fixture(); vi.stubEnv("SANDBOX_SCREEN_NETWORK", "internal"); vi.stubEnv("HOSTNAME", "supervisor"); mocks.docker.version.mockResolvedValue({ ApiVersion: apiVersion }); mocks.docker.getContainer.mockReturnValue({ inspect: vi.fn().mockResolvedValue({ NetworkSettings: { Networks: { shared: {} } }, Mounts: [ { Type: "volume", Name: "example_appdata", Destination: process.env.DATA_DIR, Source: "/var/lib/docker/volumes/example_appdata/_data", }, ], }), }); const response = await provision(); if (apiVersion === "1.44") { expect(response.status).toBe(500); expect(mocks.docker.createContainer).not.toHaveBeenCalled(); expect(mocks.docker.createNetwork).not.toHaveBeenCalled(); } else { expect(response.status).toBe(200); expect(mocks.docker.createContainer).toHaveBeenCalledWith( expect.objectContaining({ User: "1000:1000", HostConfig: expect.objectContaining({ Mounts: [ expect.objectContaining({ Type: "volume", Source: "example_appdata", Target: "/home/rakazo", VolumeOptions: { NoCopy: true, Subpath: "homes/bot" }, }), ], }), }), ); } }, ); it("does not allocate a network for an invalid home", async () => { fixture(); expect((await provision("/invalid-home")).status).toBe(500); expect(mocks.docker.createNetwork).not.toHaveBeenCalled(); }); it("does not allocate a network when home validation fails", async () => { fixture(); mocks.assertHomeWritable.mockRejectedValue(new Error("home is not writable")); expect((await provision()).status).toBe(500); expect(mocks.docker.createNetwork).not.toHaveBeenCalled(); }); it("removes the new network on every failed container creation, then can retry", async () => { const { network } = fixture(); mocks.docker.createContainer.mockRejectedValue(new Error("container creation failed")); for (let attempt = 0; attempt < 3; attempt += 1) { const response = await provision(); expect(response.status).toBe(500); expect(await response.json()).toEqual({ error: "container creation failed" }); expect(network.remove).toHaveBeenCalledTimes(attempt + 1); expect(network.remove).toHaveBeenLastCalledWith(); } const { network: retryNetwork } = fixture(); expect((await provision()).status).toBe(200); expect(retryNetwork.remove).not.toHaveBeenCalled(); }); it("removes a failed new container before its new network", async () => { const { network, container } = fixture(); container.start.mockRejectedValue(new Error("container start failed")); const response = await provision(); expect(response.status).toBe(500); expect(await response.json()).toEqual({ error: "container start failed" }); expect(container.remove).toHaveBeenCalledExactlyOnceWith(); expect(network.remove).toHaveBeenCalledExactlyOnceWith(); expect(container.remove.mock.invocationCallOrder[0]).toBeLessThan( network.remove.mock.invocationCallOrder[0]!, ); }); it("preserves the existing computer when its replacement network cannot be allocated", async () => { fixture(); const existing = { id: "existing-computer", remove: vi.fn().mockResolvedValue(undefined), inspect: vi.fn().mockResolvedValue({ Image: "old-image", Config: { Labels: { "rakazo.managed": "true", "rakazo.botId": "bot", "rakazo.spaceId": "space" }, }, HostConfig: { PortBindings: {} }, }), }; mocks.docker.listContainers.mockResolvedValue([{ Id: existing.id }]); mocks.docker.getContainer.mockReturnValue(existing); mocks.docker.createNetwork.mockRejectedValue(new Error("address pools exhausted")); const response = await provision(); expect(response.status).toBe(500); expect(await response.json()).toEqual({ error: "address pools exhausted" }); expect(existing.remove).not.toHaveBeenCalled(); expect(mocks.docker.createContainer).not.toHaveBeenCalled(); }); it("does not remove an existing network after failed creation", async () => { const { network } = fixture(); mocks.docker.createNetwork.mockRejectedValue(new Error("network already exists")); mocks.docker.createContainer.mockRejectedValue(new Error("container creation failed")); expect((await provision()).status).toBe(500); expect(network.remove).not.toHaveBeenCalled(); }); it("preserves the provision error if Docker refuses cleanup of active resources", async () => { const { network, container } = fixture(); container.start.mockRejectedValue(new Error("start response lost")); container.remove.mockRejectedValue(new Error("container is running")); network.remove.mockRejectedValue(new Error("network has active endpoints")); const response = await provision(); expect(response.status).toBe(500); expect(await response.json()).toEqual({ error: "start response lost" }); expect(container.remove).toHaveBeenCalledExactlyOnceWith(); expect(network.remove).toHaveBeenCalledExactlyOnceWith(); }); it("does not allocate or delete the shared internal network", async () => { fixture(); vi.stubEnv("SANDBOX_SCREEN_NETWORK", "internal"); vi.stubEnv("HOSTNAME", "supervisor"); mocks.docker.getContainer.mockReturnValue({ inspect: vi.fn().mockResolvedValue({ NetworkSettings: { Networks: { shared: {} } }, Mounts: [], }), }); mocks.docker.createContainer.mockRejectedValue(new Error("container creation failed")); const response = await provision(); expect(response.status).toBe(500); expect(await response.json()).toEqual({ error: "container creation failed" }); expect(mocks.docker.createContainer).toHaveBeenCalledOnce(); expect(mocks.docker.createNetwork).not.toHaveBeenCalled(); }); }); describe("space computer limit enforcement", () => { function setupContainerFixture() { const network = { remove: vi.fn().mockResolvedValue(undefined) }; const container = { id: "new-container-id", start: vi.fn().mockResolvedValue(undefined), inspect: vi.fn().mockResolvedValue({ State: { Running: true }, HostConfig: { PortBindings: {} }, }), remove: vi.fn().mockResolvedValue(undefined), }; mocks.docker.getImage.mockReturnValue({ inspect: vi.fn().mockResolvedValue({ Id: "image" }) }); mocks.docker.createNetwork.mockResolvedValue(network); mocks.docker.createContainer.mockResolvedValue(container); return { network, container }; } async function provisionBot( botId = "bot-new", spaceId = "space-1", homePath = path.join(process.env.DATA_DIR!, "homes", botId), ) { const { supervisorApp } = await import("./index.js"); return supervisorApp.request("/computers", { method: "POST", headers: { authorization: `Bearer ${resolveSupervisorToken(process.env)}`, "content-type": "application/json", "x-rakazo-bot-id": botId, "x-rakazo-space-id": spaceId, }, body: JSON.stringify({ botId, spaceId, homePath }), }); } it("rejects new container creation with 429 when space container limit is reached", async () => { setupContainerFixture(); vi.stubEnv("SANDBOX_MAX_COMPUTERS_PER_SPACE", "2"); mocks.docker.listContainers.mockImplementation( async (opts?: { filters?: { label?: string[] } }) => { const labels = opts?.filters?.label ?? []; // For findBotContainer check if (labels.some((l: string) => l.startsWith("rakazo.botId="))) { return []; } // For countSpaceContainers return [ { Id: "c1", Labels: { "rakazo.managed": "true", "rakazo.spaceId": "space-1" } }, { Id: "c2", Labels: { "rakazo.managed": "true", "rakazo.spaceId": "space-1" } }, ]; }, ); const response = await provisionBot("bot-new", "space-1"); expect(response.status).toBe(429); expect(await response.json()).toEqual({ error: "Computer limit reached for space (max: 2)", }); expect(mocks.docker.createContainer).not.toHaveBeenCalled(); }); it("allows new container creation when under space limit", async () => { setupContainerFixture(); vi.stubEnv("SANDBOX_MAX_COMPUTERS_PER_SPACE", "2"); mocks.docker.listContainers.mockImplementation( async (opts?: { filters?: { label?: string[] } }) => { const labels = opts?.filters?.label ?? []; if (labels.some((l: string) => l.startsWith("rakazo.botId="))) { return []; } return [{ Id: "c1", Labels: { "rakazo.managed": "true", "rakazo.spaceId": "space-1" } }]; }, ); const response = await provisionBot("bot-new", "space-1"); expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ id: "new-container-id", }); expect(mocks.docker.createContainer).toHaveBeenCalledOnce(); }); it("resumes existing container even if space is at limit", async () => { setupContainerFixture(); vi.stubEnv("SANDBOX_MAX_COMPUTERS_PER_SPACE", "1"); const existing = { id: "existing-container", inspect: vi.fn().mockResolvedValue({ Image: "image", Config: { User: hostComputerUser(process.getuid?.(), process.getgid?.()), Labels: { "rakazo.managed": "true", "rakazo.botId": "bot-existing", "rakazo.spaceId": "space-1", }, }, State: { Running: true }, HostConfig: { NetworkMode: computerNetworkNameFor("bot-existing"), PortBindings: {}, Mounts: [], }, }), start: vi.fn().mockResolvedValue(undefined), }; mocks.docker.getContainer.mockReturnValue(existing); mocks.docker.listContainers.mockImplementation( async (opts?: { filters?: { label?: string[] } }) => { const labels = opts?.filters?.label ?? []; if (labels.some((l: string) => l === "rakazo.botId=bot-existing")) { return [ { Id: existing.id, Labels: { "rakazo.managed": "true", "rakazo.botId": "bot-existing", "rakazo.spaceId": "space-1", }, }, ]; } return [ { Id: existing.id, Labels: { "rakazo.managed": "true", "rakazo.spaceId": "space-1" } }, ]; }, ); const response = await provisionBot("bot-existing", "space-1"); expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ id: "existing-container", resumed: true, }); expect(mocks.docker.createContainer).not.toHaveBeenCalled(); }); it("counts legacy workspaceId COMPUTER_IMAGE containers toward the limit", async () => { setupContainerFixture(); vi.stubEnv("SANDBOX_MAX_COMPUTERS_PER_SPACE", "1"); mocks.docker.listContainers.mockImplementation( async (opts?: { filters?: { label?: string[] } }) => { const labels = opts?.filters?.label ?? []; if (labels.some((l: string) => l.startsWith("rakazo.botId="))) { return []; } // Legacy managed computer: COMPUTER_IMAGE + workspaceId, no rakazo.managed. return [ { Id: "legacy", Image: COMPUTER_IMAGE, Labels: { "rakazo.workspaceId": "space-1", "rakazo.botId": "legacy-bot" }, }, ]; }, ); const response = await provisionBot("bot-new", "space-1"); expect(response.status).toBe(429); expect(await response.json()).toEqual({ error: "Computer limit reached for space (max: 1)", }); expect(mocks.docker.createContainer).not.toHaveBeenCalled(); }); it("serializes concurrent creates for different bots in the same space", async () => { const { container } = setupContainerFixture(); vi.stubEnv("SANDBOX_MAX_COMPUTERS_PER_SPACE", "1"); let created = 0; mocks.docker.listContainers.mockImplementation( async (opts?: { filters?: { label?: string[] } }) => { const labels = opts?.filters?.label ?? []; if (labels.some((l: string) => l.startsWith("rakazo.botId="))) { return []; } return Array.from({ length: created }, (_, index) => ({ Id: `c${index}`, Labels: { "rakazo.managed": "true", "rakazo.spaceId": "space-1" }, })); }, ); mocks.docker.createContainer.mockImplementation(async () => { await new Promise((resolve) => setTimeout(resolve, 40)); created += 1; return { ...container, id: `new-container-${created}`, }; }); const [first, second] = await Promise.all([ provisionBot("bot-a", "space-1"), provisionBot("bot-b", "space-1"), ]); const statuses = [first.status, second.status].sort((a, b) => a - b); expect(statuses).toEqual([200, 429]); expect(mocks.docker.createContainer).toHaveBeenCalledOnce(); const rejected = first.status === 429 ? first : second; expect(await rejected.json()).toEqual({ error: "Computer limit reached for space (max: 1)", }); }); it("serializes incompatible replace with a concurrent fresh create at the cap", async () => { const { container } = setupContainerFixture(); vi.stubEnv("SANDBOX_MAX_COMPUTERS_PER_SPACE", "1"); const present = new Set(["existing-incompatible"]); const existing = { id: "existing-incompatible", inspect: vi.fn().mockResolvedValue({ Image: "stale-image", Config: { User: hostComputerUser(process.getuid?.(), process.getgid?.()), Labels: { "rakazo.managed": "true", "rakazo.botId": "bot-existing", "rakazo.spaceId": "space-1", }, }, State: { Running: true }, HostConfig: { NetworkMode: computerNetworkNameFor("bot-existing"), PortBindings: {}, Mounts: [], }, }), start: vi.fn().mockResolvedValue(undefined), remove: vi.fn(async () => { await new Promise((resolve) => setTimeout(resolve, 40)); present.delete("existing-incompatible"); }), }; mocks.docker.getContainer.mockImplementation((id: string) => id === existing.id ? existing : container, ); mocks.docker.listContainers.mockImplementation( async (opts?: { filters?: { label?: string[] } }) => { const labels = opts?.filters?.label ?? []; if (labels.some((l: string) => l === "rakazo.botId=bot-existing")) { return present.has(existing.id) ? [ { Id: existing.id, Labels: { "rakazo.managed": "true", "rakazo.botId": "bot-existing", "rakazo.spaceId": "space-1", }, }, ] : []; } if (labels.some((l: string) => l.startsWith("rakazo.botId="))) { return []; } return [...present].map((Id) => ({ Id, Labels: { "rakazo.managed": "true", "rakazo.spaceId": "space-1" }, })); }, ); mocks.docker.createContainer.mockImplementation(async () => { await new Promise((resolve) => setTimeout(resolve, 40)); const id = `created-${present.size + 1}`; present.add(id); return { ...container, id }; }); const [replaceResponse, createResponse] = await Promise.all([ provisionBot("bot-existing", "space-1"), provisionBot("bot-new", "space-1"), ]); // With the space lock, the incompatible replace keeps its slot and must succeed; // the concurrent fresh create must see the space still at capacity. expect(replaceResponse.status).toBe(200); expect(createResponse.status).toBe(429); expect(mocks.docker.createContainer).toHaveBeenCalledOnce(); expect(present.size).toBe(1); expect(await createResponse.json()).toEqual({ error: "Computer limit reached for space (max: 1)", }); }); });