6c7eed7580
Fresh history cut from cynarAI/evoo antigravity-handoff/project-timeline (Voice First + project timeline). Brand docs/titles as evOo; keep @rakazo/* packages as tech debt. Excludes node_modules, caches, secrets, voice blobs.
324 lines
14 KiB
TypeScript
324 lines
14 KiB
TypeScript
import { mkdtemp, rm } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import type { AdapterContext, JobPublisher, SandboxProvider } from "@rakazo/adapter-kit";
|
|
import type { PrismaClient, ThreadEvents } from "@rakazo/db";
|
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import { ComputerBusyError, provisionComputer, replaceComputer } from "./computer-lifecycle.js";
|
|
import { DesktopSandboxProvider } from "./desktop-sandbox.js";
|
|
import { FakeSandboxProvider } from "./fake-sandbox.js";
|
|
import { LocalAgentHomeStore } from "./home.js";
|
|
|
|
const context = {
|
|
operationId: "recovery",
|
|
traceId: "recovery",
|
|
spaceId: "space",
|
|
userId: "user",
|
|
botId: "bot",
|
|
signal: new AbortController().signal,
|
|
} satisfies AdapterContext;
|
|
const roots: string[] = [];
|
|
afterEach(async () => {
|
|
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
|
});
|
|
|
|
async function fixture(provider: "fake" | "desktop" = "fake") {
|
|
const root = await mkdtemp(path.join(tmpdir(), "rakazo-recovery-"));
|
|
roots.push(root);
|
|
const sandbox: SandboxProvider =
|
|
provider === "desktop" ? new DesktopSandboxProvider({ root }) : new FakeSandboxProvider();
|
|
const home = new LocalAgentHomeStore(path.join(root, "homes"));
|
|
const first = await sandbox.provision({ botId: "bot", homePath: root }, context);
|
|
await home.writeFile("bot", "notes.txt", "checkpoint", context);
|
|
await sandbox.writeFile(
|
|
first,
|
|
{ path: "notes.txt", content: new TextEncoder().encode("live work") },
|
|
context,
|
|
);
|
|
const row = {
|
|
id: "computer",
|
|
homeKey: "bot",
|
|
scope: "dedicated",
|
|
state: "running",
|
|
providerRef: first.providerRef as string | null,
|
|
kind: first.kind,
|
|
controlHolder: "none",
|
|
controlLeaseId: null,
|
|
maintenanceId: null as string | null,
|
|
homeRevision: "saved",
|
|
updatedAt: new Date("2024-01-01T00:00:00.000Z"),
|
|
};
|
|
// Model the atomic scalar predicates used by claims, including stale references
|
|
// and the updatedAt claim stamp used for booting reclaim / end-of-boot writes.
|
|
const computer = {
|
|
findUniqueOrThrow: vi.fn(async () => ({ ...row })),
|
|
updateMany: vi.fn(async ({ where, data }) => {
|
|
const matches = ["id", "state", "providerRef", "kind", "updatedAt"].every(
|
|
(key) => !(key in where) || where[key] === row[key as keyof typeof row],
|
|
);
|
|
if (!matches) return { count: 0 };
|
|
Object.assign(row, { updatedAt: new Date(row.updatedAt.getTime() + 1) }, data);
|
|
return { count: 1 };
|
|
}),
|
|
update: vi.fn(async ({ data }) =>
|
|
Object.assign(row, { updatedAt: new Date(row.updatedAt.getTime() + 1) }, data),
|
|
),
|
|
};
|
|
const deps = {
|
|
prisma: {
|
|
computer,
|
|
computerExecutionLease: { findFirst: vi.fn().mockResolvedValue(null) },
|
|
run: { findFirst: vi.fn().mockResolvedValue(null) },
|
|
} as unknown as PrismaClient,
|
|
sandbox,
|
|
home,
|
|
jobs: {} as JobPublisher,
|
|
events: {} as ThreadEvents,
|
|
dataDir: root,
|
|
};
|
|
return { deps, row, computer, first, root };
|
|
}
|
|
|
|
describe("computer recovery preserves live work", () => {
|
|
it("blocks competing maintenance and boots while an update owns the computer", async () => {
|
|
const { deps, row } = await fixture();
|
|
row.maintenanceId = "other-update";
|
|
const destroy = vi.spyOn(deps.sandbox, "destroy");
|
|
const provision = vi.spyOn(deps.sandbox, "provision");
|
|
await expect(provisionComputer(deps, row.id, context)).rejects.toBeInstanceOf(
|
|
ComputerBusyError,
|
|
);
|
|
for (const mode of ["update", "recover", "reset"] as const) {
|
|
await expect(replaceComputer(deps, row.id, mode, context)).rejects.toBeInstanceOf(
|
|
ComputerBusyError,
|
|
);
|
|
}
|
|
expect(destroy).not.toHaveBeenCalled();
|
|
expect(provision).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("claims a running computer before concurrent workers can create competing replacements", async () => {
|
|
const { deps, row, computer, first } = await fixture();
|
|
await deps.sandbox.destroy(first, context);
|
|
row.providerRef = "missing-provider";
|
|
// Both workers read the original row before either claims it; later reads
|
|
// (claim stamp, activation) must see the live row so updatedAt fencing works.
|
|
const original = { ...row };
|
|
let reads = 0;
|
|
computer.findUniqueOrThrow.mockImplementation(async () => {
|
|
reads += 1;
|
|
if (reads <= 2) return { ...original };
|
|
return { ...row };
|
|
});
|
|
const provision = vi.spyOn(deps.sandbox, "provision");
|
|
const restore = vi.spyOn(deps.sandbox, "importWorkspace");
|
|
const results = await Promise.allSettled([
|
|
provisionComputer(deps, row.id, context),
|
|
provisionComputer(deps, row.id, { ...context, botId: "other-bot" }),
|
|
]);
|
|
const successful = results.filter((result) => result.status === "fulfilled");
|
|
const rejected = results.filter((result) => result.status === "rejected");
|
|
expect(successful).toHaveLength(1);
|
|
expect(rejected).toHaveLength(1);
|
|
expect(rejected[0]?.reason).toBeInstanceOf(ComputerBusyError);
|
|
expect(provision).toHaveBeenCalledOnce();
|
|
expect(restore).toHaveBeenCalledOnce();
|
|
expect(row.providerRef).toBe(successful[0]?.value.providerRef);
|
|
expect(row.state).toBe("running");
|
|
});
|
|
|
|
it("rejects a delayed reconnect whose original reference was already replaced", async () => {
|
|
const { deps, row, computer, first } = await fixture();
|
|
await deps.sandbox.destroy(first, context);
|
|
row.providerRef = "missing-provider";
|
|
const stale = { ...row };
|
|
const winner = await provisionComputer(deps, row.id, context);
|
|
await deps.sandbox.writeFile(
|
|
winner,
|
|
{ path: "notes.txt", content: new TextEncoder().encode("winner work") },
|
|
context,
|
|
);
|
|
computer.findUniqueOrThrow.mockResolvedValue(stale);
|
|
const provision = vi.spyOn(deps.sandbox, "provision");
|
|
await expect(provisionComputer(deps, row.id, context)).rejects.toBeInstanceOf(
|
|
ComputerBusyError,
|
|
);
|
|
expect(provision).not.toHaveBeenCalled();
|
|
expect(row.providerRef).toBe(winner.providerRef);
|
|
expect(
|
|
new TextDecoder().decode(await deps.sandbox.readFile(winner, "notes.txt", context)),
|
|
).toBe("winner work");
|
|
});
|
|
|
|
it("claims an abandoned booting computer only once when callers have no screenLeaseId", async () => {
|
|
const { deps, row, computer } = await fixture();
|
|
row.state = "booting";
|
|
row.providerRef = null;
|
|
const observed = { ...row };
|
|
let reads = 0;
|
|
computer.findUniqueOrThrow.mockImplementation(async () => {
|
|
reads += 1;
|
|
// First two reads are the concurrent observations before either claims.
|
|
if (reads <= 2) return { ...observed };
|
|
return { ...row };
|
|
});
|
|
const provision = vi.spyOn(deps.sandbox, "provision");
|
|
const setTimeoutReal = globalThis.setTimeout;
|
|
vi.stubGlobal("setTimeout", ((fn: (...args: never[]) => void, _ms?: number, ...args: never[]) =>
|
|
setTimeoutReal(fn, 0, ...args)) as unknown as typeof setTimeout);
|
|
try {
|
|
const results = await Promise.allSettled([
|
|
provisionComputer(deps, row.id, context),
|
|
provisionComputer(deps, row.id, { ...context, botId: "other-bot" }),
|
|
]);
|
|
const successful = results.filter((result) => result.status === "fulfilled");
|
|
const rejected = results.filter((result) => result.status === "rejected");
|
|
expect(successful).toHaveLength(1);
|
|
expect(rejected).toHaveLength(1);
|
|
expect(rejected[0]?.reason).toBeInstanceOf(ComputerBusyError);
|
|
expect(provision).toHaveBeenCalledOnce();
|
|
expect(row.state).toBe("running");
|
|
} finally {
|
|
vi.unstubAllGlobals();
|
|
}
|
|
});
|
|
|
|
it("preserves the reference and workspace through an uncertain reconnect and a retry", async () => {
|
|
const { deps, row, first } = await fixture();
|
|
const error = new Error("fetch failed");
|
|
vi.spyOn(deps.sandbox, "provision").mockRejectedValueOnce(error);
|
|
const restore = vi.spyOn(deps.sandbox, "importWorkspace");
|
|
const destroy = vi.spyOn(deps.sandbox, "destroy");
|
|
await expect(provisionComputer(deps, row.id, context)).rejects.toBe(error);
|
|
expect(row).toMatchObject({ state: "running", providerRef: first.providerRef });
|
|
const reconnected = await provisionComputer(deps, row.id, context);
|
|
expect(reconnected).toMatchObject({ providerRef: first.providerRef, fresh: false });
|
|
expect(restore).not.toHaveBeenCalled();
|
|
expect(destroy).not.toHaveBeenCalled();
|
|
expect(
|
|
new TextDecoder().decode(await deps.sandbox.readFile(reconnected, "notes.txt", context)),
|
|
).toBe("live work");
|
|
});
|
|
|
|
it.each(["running", "error", "stopped", "suspended"])(
|
|
"reconnects desktop files after an adapter restart from %s without importing an old checkpoint",
|
|
async (state) => {
|
|
const { deps, row, root, first } = await fixture("desktop");
|
|
row.state = state;
|
|
deps.sandbox = new DesktopSandboxProvider({ root });
|
|
const restore = vi.spyOn(deps.sandbox, "importWorkspace");
|
|
const reconnected = await provisionComputer(deps, row.id, context);
|
|
expect(reconnected).toMatchObject({ providerRef: first.providerRef, fresh: false });
|
|
expect(restore).not.toHaveBeenCalled();
|
|
expect(
|
|
new TextDecoder().decode(await deps.sandbox.readFile(reconnected, "notes.txt", context)),
|
|
).toBe("live work");
|
|
},
|
|
);
|
|
|
|
it("restores the checkpoint when a desktop workspace is actually absent after restart", async () => {
|
|
const { deps, row, root, first } = await fixture("desktop");
|
|
expect(first.fresh).toBe(true);
|
|
await rm(first.providerRef, { recursive: true });
|
|
deps.sandbox = new DesktopSandboxProvider({ root });
|
|
const reconnected = await provisionComputer(deps, row.id, context);
|
|
expect(reconnected).toMatchObject({ providerRef: first.providerRef, fresh: true });
|
|
expect(
|
|
new TextDecoder().decode(await deps.sandbox.readFile(reconnected, "notes.txt", context)),
|
|
).toBe("checkpoint");
|
|
});
|
|
|
|
it.each([
|
|
"Path notes.txt not found",
|
|
"404: file not found",
|
|
"command not found",
|
|
"checkpoint directory does not exist",
|
|
"export process killed",
|
|
"ECONNRESET",
|
|
"Sandbox not found",
|
|
])("aborts update and its retry when checkpoint fails with %s", async (message) => {
|
|
const { deps, row, first } = await fixture();
|
|
const error = new Error(message);
|
|
vi.spyOn(deps.sandbox, "exportWorkspace").mockImplementation(async function* () {
|
|
yield { path: "notes.txt", content: new TextEncoder().encode("incomplete checkpoint") };
|
|
throw error;
|
|
});
|
|
const destroy = vi.spyOn(deps.sandbox, "destroy");
|
|
const provision = vi.spyOn(deps.sandbox, "provision");
|
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
await expect(replaceComputer(deps, row.id, "update", context)).rejects.toBe(error);
|
|
expect(row).toMatchObject({
|
|
state: "error",
|
|
providerRef: first.providerRef,
|
|
homeRevision: "saved",
|
|
});
|
|
}
|
|
expect(destroy).not.toHaveBeenCalled();
|
|
expect(provision).not.toHaveBeenCalled();
|
|
expect(new TextDecoder().decode(await deps.sandbox.readFile(first, "notes.txt", context))).toBe(
|
|
"live work",
|
|
);
|
|
expect(await deps.home.readFile("bot", "notes.txt", context)).toBe("checkpoint");
|
|
});
|
|
|
|
it.each(["running", "error"])(
|
|
"updates a %s computer only after saving its current workspace",
|
|
async (state) => {
|
|
const { deps, row } = await fixture();
|
|
row.state = state;
|
|
const updated = await replaceComputer(deps, row.id, "update", context);
|
|
expect(updated.fresh).toBe(true);
|
|
expect(row).toMatchObject({ state: "running", providerRef: updated.providerRef });
|
|
expect(
|
|
new TextDecoder().decode(await deps.sandbox.readFile(updated, "notes.txt", context)),
|
|
).toBe("live work");
|
|
expect(await deps.home.readFile("bot", "notes.txt", context)).toBe("live work");
|
|
},
|
|
);
|
|
|
|
it("resets a computer stuck suspending after a hung stop", async () => {
|
|
const { deps, row } = await fixture();
|
|
row.state = "suspending";
|
|
const updated = await replaceComputer(deps, row.id, "reset", context);
|
|
expect(updated.fresh).toBe(true);
|
|
expect(row).toMatchObject({ state: "running", providerRef: updated.providerRef });
|
|
expect(
|
|
new TextDecoder().decode(await deps.sandbox.readFile(updated, "notes.txt", context)),
|
|
).toBe("checkpoint");
|
|
});
|
|
|
|
it("refuses Reset while a suspend claim is still live", async () => {
|
|
const { deps, row, computer } = await fixture();
|
|
row.state = "suspending";
|
|
row.updatedAt = new Date();
|
|
const destroy = vi.spyOn(deps.sandbox, "destroy");
|
|
await expect(replaceComputer(deps, row.id, "reset", context)).rejects.toBeInstanceOf(
|
|
ComputerBusyError,
|
|
);
|
|
expect(destroy).not.toHaveBeenCalled();
|
|
expect(computer.updateMany).not.toHaveBeenCalled();
|
|
expect(row.state).toBe("suspending");
|
|
});
|
|
|
|
it("resets a missing computer after idempotent provider teardown", async () => {
|
|
const { deps, row, first } = await fixture();
|
|
await deps.sandbox.destroy(first, context);
|
|
const updated = await replaceComputer(deps, row.id, "reset", context);
|
|
expect(updated.fresh).toBe(true);
|
|
expect(row).toMatchObject({ state: "running", providerRef: updated.providerRef });
|
|
expect(
|
|
new TextDecoder().decode(await deps.sandbox.readFile(updated, "notes.txt", context)),
|
|
).toBe("checkpoint");
|
|
});
|
|
|
|
it("preserves the reference when reset teardown fails ambiguously", async () => {
|
|
const { deps, row, first } = await fixture();
|
|
const failure = new Error("404: configuration file not found");
|
|
vi.spyOn(deps.sandbox, "destroy").mockRejectedValue(failure);
|
|
const provision = vi.spyOn(deps.sandbox, "provision");
|
|
await expect(replaceComputer(deps, row.id, "reset", context)).rejects.toBe(failure);
|
|
expect(provision).not.toHaveBeenCalled();
|
|
expect(row.providerRef).toBe(first.providerRef);
|
|
});
|
|
});
|