Refresh a containerized sandbox supervisor on deploy (#215)
A supervisor that runs as a standalone container kept the image it was started with, so broker fixes never reached it. The deploy now rebuilds that image when its sources change and recreates the container with the same network, restart policy, mounts and environment, restoring the old one if the new one stays unhealthy. The broker socket moves into the data directory and is shared from containerized supervisors too. Claude-Session: https://claude.ai/code/session_01Tgvrj34NMnaqvtgvruYQZZ Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
cynarAI
GitHub
parent
9bc3187a4a
commit
a0bdf6ddc8
@ -203,10 +203,9 @@ describe("sandbox supervisor HTTP boundary", () => {
|
||||
expect(needsBridgeBrokerListener("0.0.0.0", "linux", false)).toBe(false);
|
||||
});
|
||||
|
||||
it("shares the GitHub broker socket from any Linux host supervisor", () => {
|
||||
expect(servesGithubBrokerSocket("linux", false)).toBe(true);
|
||||
expect(servesGithubBrokerSocket("linux", true)).toBe(false);
|
||||
expect(servesGithubBrokerSocket("darwin", false)).toBe(false);
|
||||
it("shares the GitHub broker socket from any Linux supervisor, containerized or not", () => {
|
||||
expect(servesGithubBrokerSocket("linux")).toBe(true);
|
||||
expect(servesGithubBrokerSocket("darwin")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps the larger file-write allowance scoped to that exact POST route", () => {
|
||||
|
||||
@ -2,7 +2,6 @@ import { randomUUID } from "node:crypto";
|
||||
import { chmodSync, existsSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
@ -158,18 +157,15 @@ export function needsBridgeBrokerListener(
|
||||
// run as long; Node's 5-minute default would cut them.
|
||||
const serverOptions = { requestTimeout: 2 * 60 * 60 * 1000 };
|
||||
|
||||
// Docker binds host paths, so only a supervisor running directly on a Linux host can share one.
|
||||
export function servesGithubBrokerSocket(
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
inContainer = existsSync("/.dockerenv"),
|
||||
): boolean {
|
||||
return platform === "linux" && !inContainer;
|
||||
// Docker Desktop cannot pass a host unix socket into a container, so only Linux shares one.
|
||||
export function servesGithubBrokerSocket(platform: NodeJS.Platform = process.platform): boolean {
|
||||
return platform === "linux";
|
||||
}
|
||||
|
||||
// A socket bind-mounted into each computer reaches the broker without crossing the host firewall.
|
||||
// It lives in the data directory, which a containerized supervisor already shares with the host.
|
||||
const githubBrokerSocketDir = path.resolve(
|
||||
process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ??
|
||||
path.join(os.homedir(), ".rakazo", "github-broker"),
|
||||
process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ?? path.join(dataDir, "github-broker"),
|
||||
);
|
||||
// Set once the socket listens, so computers only get the mount when there is something behind it.
|
||||
let githubBrokerSocketReady = false;
|
||||
@ -191,6 +187,14 @@ function listenGithubBrokerSocket(logger: ReturnType<typeof createRootLogger>) {
|
||||
return server;
|
||||
}
|
||||
|
||||
// Where Docker finds the socket directory on the host. A data volume has no host path to bind,
|
||||
// so those computers keep reaching the broker over the network.
|
||||
function githubBrokerSocketHostDir(runtimeInfo: Docker.ContainerInspectInfo | undefined) {
|
||||
if (!githubBrokerSocketReady) return undefined;
|
||||
const storage = computerHomeStorage(githubBrokerSocketDir, dataDir, runtimeInfo);
|
||||
return storage.homeVolume ? undefined : storage.homePath;
|
||||
}
|
||||
|
||||
async function bridgeGatewayAddress(): Promise<string | undefined> {
|
||||
const network = await docker.getNetwork("bridge").inspect();
|
||||
return network.IPAM?.Config?.find((entry: { Gateway?: string }) => entry.Gateway)?.Gateway;
|
||||
@ -256,6 +260,7 @@ app.post("/computers", async (c) => {
|
||||
assertVolumeSubpathSupport((await docker.version()).ApiVersion);
|
||||
}
|
||||
const computerUser = runtimeInfo ? COMPUTER_USER : hostComputerUser(hostUid, hostGid);
|
||||
const brokerSocketDir = githubBrokerSocketHostDir(runtimeInfo);
|
||||
const existing = await findBotContainer(body.botId, body.spaceId);
|
||||
if (existing) {
|
||||
const info = await existing.inspect();
|
||||
@ -270,7 +275,7 @@ app.post("/computers", async (c) => {
|
||||
info.Config.User === computerUser &&
|
||||
controlPublishOk &&
|
||||
(!storage.homeVolume || homeVolumeMatches(info.HostConfig.Mounts, storage.homeVolume)) &&
|
||||
(!githubBrokerSocketReady || githubBrokerSocketMounted(info.Mounts))
|
||||
(!brokerSocketDir || githubBrokerSocketMounted(info.Mounts))
|
||||
) {
|
||||
if (!info.State.Running) await existing.start();
|
||||
return c.json({
|
||||
@ -333,7 +338,7 @@ app.post("/computers", async (c) => {
|
||||
networkMode,
|
||||
controlToken: randomUUID(),
|
||||
publishControlPort: controlViaLoopback,
|
||||
...(githubBrokerSocketReady ? { githubBrokerSocketDir } : {}),
|
||||
...(brokerSocketDir ? { githubBrokerSocketDir: brokerSocketDir } : {}),
|
||||
}),
|
||||
);
|
||||
await container.start();
|
||||
|
||||
@ -176,6 +176,8 @@ Do not commit host passwords, tokens, or SSH private keys. Configure restart tar
|
||||
| `EVO_API_HEALTH_URL` | `http://127.0.0.1:3100/health` | API health probe |
|
||||
| `EVO_WEB_HEALTH_URL` | `http://127.0.0.1:5173/` | Web health probe |
|
||||
| `EVO_HEALTH_TIMEOUT_SEC` | `120` | Per-check timeout (seconds) |
|
||||
| `EVO_SUPERVISOR_CONTAINER` | `evoo-sandbox-supervisor` | Standalone supervisor container rebuilt and recreated with its settings when its sources change |
|
||||
| `EVO_SUPERVISOR_HEALTH_URL` | `http://127.0.0.1:7091/health` | Supervisor health probe; an unhealthy new container is replaced by the previous one |
|
||||
|
||||
`start-stack.sh`:
|
||||
|
||||
|
||||
@ -480,6 +480,73 @@ WHERE "state" = 'stopped'
|
||||
SQL
|
||||
}
|
||||
|
||||
# A host that runs the sandbox supervisor as a standalone container keeps the image it was started
|
||||
# with. Rebuild it when its sources changed and start a fresh container with the same name, image,
|
||||
# network, restart policy, mounts and environment; the old one comes back if the new one is unhealthy.
|
||||
refresh_supervisor_container() {
|
||||
local previous_sha="$1" target_sha="$2"
|
||||
local name="${EVO_SUPERVISOR_CONTAINER:-evoo-sandbox-supervisor}"
|
||||
local health_url="${EVO_SUPERVISOR_HEALTH_URL:-http://127.0.0.1:7091/health}"
|
||||
git -C "$EVO_DEPLOY_DIR" diff --quiet "$previous_sha" "$target_sha" -- \
|
||||
infra/sandboxes/supervisor packages/contracts packages/core packages/logging pnpm-lock.yaml && return 0
|
||||
command -v docker >/dev/null 2>&1 || return 0
|
||||
docker container inspect "$name" >/dev/null 2>&1 || return 0
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "[dry-run] would rebuild and recreate the $name container"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local work image network restart retries line
|
||||
work="$(mktemp -d)"
|
||||
image="$(docker container inspect -f '{{.Config.Image}}' "$name")"
|
||||
network="$(docker container inspect -f '{{.HostConfig.NetworkMode}}' "$name")"
|
||||
restart="$(docker container inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$name")"
|
||||
retries="$(docker container inspect -f '{{.HostConfig.RestartPolicy.MaximumRetryCount}}' "$name")"
|
||||
local -a args=()
|
||||
[[ -n "$network" && "$network" != default && "$network" != bridge ]] && args+=(--network "$network")
|
||||
if [[ "$restart" == on-failure && "$retries" != 0 ]]; then
|
||||
args+=(--restart "on-failure:$retries")
|
||||
elif [[ -n "$restart" && "$restart" != no ]]; then
|
||||
args+=(--restart "$restart")
|
||||
fi
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && args+=(-v "$line")
|
||||
done < <(docker container inspect -f \
|
||||
'{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}{{else}}{{.Source}}{{end}}:{{.Destination}}{{if not .RW}}:ro{{end}}{{"\n"}}{{end}}' "$name")
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && args+=(-p "${line#:}")
|
||||
done < <(docker container inspect -f \
|
||||
'{{range $port, $bindings := .HostConfig.PortBindings}}{{range $bindings}}{{.HostIp}}:{{.HostPort}}:{{$port}}{{"\n"}}{{end}}{{end}}' "$name")
|
||||
# The image sets PATH and its Node versions itself.
|
||||
(umask 077 && docker container inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$name" |
|
||||
grep -Ev '^(PATH|NODE_VERSION|YARN_VERSION)=' >"$work/env") || true
|
||||
args+=(--env-file "$work/env")
|
||||
|
||||
log "rebuilding the $name container"
|
||||
if ! docker build -q -t "$image" -f "$EVO_DEPLOY_DIR/infra/sandboxes/supervisor/Dockerfile" "$EVO_DEPLOY_DIR" >&2; then
|
||||
log "WARNING: the supervisor image rebuild failed; $name keeps running the old image"
|
||||
rm -rf "$work"
|
||||
return 0
|
||||
fi
|
||||
docker rm -f "$name-previous" >/dev/null 2>&1 || true
|
||||
if ! { docker stop "$name" >/dev/null && docker rename "$name" "$name-previous"; }; then
|
||||
log "WARNING: could not stop $name; it keeps running the old image"
|
||||
docker start "$name" >/dev/null 2>&1 || true
|
||||
rm -rf "$work"
|
||||
return 0
|
||||
fi
|
||||
if docker run -d --name "$name" "${args[@]}" "$image" >/dev/null && wait_for_url "$health_url" "supervisor"; then
|
||||
docker rm "$name-previous" >/dev/null
|
||||
log "$name now runs the new image"
|
||||
else
|
||||
log "WARNING: the new $name container did not come up; restoring the previous one"
|
||||
docker rm -f "$name" >/dev/null 2>&1 || true
|
||||
{ docker rename "$name-previous" "$name" && docker start "$name" >/dev/null; } ||
|
||||
log "WARNING: could not restore $name; start it with docker start $name-previous"
|
||||
fi
|
||||
rm -rf "$work"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
validate_deploy_dir
|
||||
@ -509,6 +576,7 @@ main() {
|
||||
|
||||
write_state "$previous_sha" "$target_sha" "success"
|
||||
rebuild_computer_image "$previous_sha" "$target_sha"
|
||||
refresh_supervisor_container "$previous_sha" "$target_sha"
|
||||
recreate_idle_computers "$previous_sha" "$target_sha"
|
||||
log "deploy succeeded: $previous_sha -> $target_sha"
|
||||
}
|
||||
|
||||
@ -49,6 +49,7 @@ interface Revision {
|
||||
build?: "ok" | "fail";
|
||||
health?: "ok" | "down";
|
||||
computer?: string;
|
||||
supervisor?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -122,6 +123,10 @@ printf 'restart %s %s\\n' "$(cat "$EVO_DEPLOY_DIR/version.txt")" "$served" >> "$
|
||||
path.join(checkout, "infra/sandboxes/computer/Dockerfile"),
|
||||
revision.computer ?? "FROM a\n",
|
||||
);
|
||||
write(
|
||||
path.join(checkout, "infra/sandboxes/supervisor/Dockerfile"),
|
||||
revision.supervisor ?? "FROM a\n",
|
||||
);
|
||||
git("add", "-A");
|
||||
git("commit", "-q", "-m", version);
|
||||
return git("rev-parse", "HEAD");
|
||||
@ -196,6 +201,53 @@ describe("evo deploy", () => {
|
||||
expect(cmds.at(-1)).toBe("B --filter @rakazo/db exec prisma db execute --stdin");
|
||||
});
|
||||
|
||||
it("recreates a supervisor container with its settings when its sources changed", () => {
|
||||
const fakeDocker = (bin: string) =>
|
||||
write(
|
||||
path.join(bin, "docker"),
|
||||
`#!/bin/bash
|
||||
if [[ "$1 $2" == "container inspect" ]]; then
|
||||
case "$4" in
|
||||
*Config.Image*) echo rakazo/sandbox-supervisor:local ;;
|
||||
*NetworkMode*) echo host ;;
|
||||
*RestartPolicy.Name*) echo unless-stopped ;;
|
||||
*MaximumRetryCount*) echo 0 ;;
|
||||
*Mounts*) printf '/srv/data:/srv/data\\n/var/run/docker.sock:/var/run/docker.sock\\n' ;;
|
||||
*Config.Env*) printf 'DATA_DIR=/srv/data\\nPATH=/usr/bin\\nSUPERVISOR_PORT=7091\\n' ;;
|
||||
esac
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$1" == run ]]; then
|
||||
for arg in "$@"; do [[ "$prev" == --env-file ]] && cp "$arg" "$(dirname "$COMMAND_LOG")/supervisor.env"; prev="$arg"; done
|
||||
fi
|
||||
echo "docker $*" | sed "s|--env-file [^ ]*|--env-file ENV|" >> "$COMMAND_LOG"
|
||||
`,
|
||||
0o755,
|
||||
);
|
||||
|
||||
const unchanged = deployFixture({});
|
||||
fakeDocker(path.join(unchanged.checkout, "../bin"));
|
||||
expect(unchanged.run().status).toBe(0);
|
||||
expect(unchanged.commands().some((line) => line.startsWith("docker build"))).toBe(false);
|
||||
|
||||
const changed = deployFixture({ supervisor: "FROM b\n" });
|
||||
fakeDocker(path.join(changed.checkout, "../bin"));
|
||||
const result = changed.run();
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const docker = changed.commands().filter((line) => line.startsWith("docker"));
|
||||
expect(docker).toEqual([
|
||||
`docker build -q -t rakazo/sandbox-supervisor:local -f ${path.join(changed.checkout, "infra/sandboxes/supervisor/Dockerfile")} ${changed.checkout}`,
|
||||
"docker rm -f evoo-sandbox-supervisor-previous",
|
||||
"docker stop evoo-sandbox-supervisor",
|
||||
"docker rename evoo-sandbox-supervisor evoo-sandbox-supervisor-previous",
|
||||
"docker run -d --name evoo-sandbox-supervisor --network host --restart unless-stopped -v /srv/data:/srv/data -v /var/run/docker.sock:/var/run/docker.sock --env-file ENV rakazo/sandbox-supervisor:local",
|
||||
"docker rm evoo-sandbox-supervisor-previous",
|
||||
]);
|
||||
expect(readFileSync(path.join(changed.checkout, "../supervisor.env"), "utf8")).toBe(
|
||||
"DATA_DIR=/srv/data\nSUPERVISOR_PORT=7091\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("stops a failing build before the restart and keeps the served build", () => {
|
||||
const f = deployFixture({ build: "fail" });
|
||||
const result = f.run();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user