From a0bdf6ddc8b8921db0973907fb899e394983ec00 Mon Sep 17 00:00:00 2001 From: cynarAI <155274210+cynarAI@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:25:09 +0200 Subject: [PATCH] Refresh a containerized sandbox supervisor on deploy (#215) A supervisor that runs as a standalone container kept the image it was started with, so broker fixes never reached it. The deploy now rebuilds that image when its sources change and recreates the container with the same network, restart policy, mounts and environment, restoring the old one if the new one stays unhealthy. The broker socket moves into the data directory and is shared from containerized supervisors too. Claude-Session: https://claude.ai/code/session_01Tgvrj34NMnaqvtgvruYQZZ Co-authored-by: Claude --- infra/sandboxes/supervisor/src/index.test.ts | 7 +- infra/sandboxes/supervisor/src/index.ts | 27 ++++---- ops/evo-deploy/README.md | 2 + ops/evo-deploy/deploy.sh | 68 ++++++++++++++++++++ packages/testkit/src/evo-deploy.test.ts | 52 +++++++++++++++ 5 files changed, 141 insertions(+), 15 deletions(-) diff --git a/infra/sandboxes/supervisor/src/index.test.ts b/infra/sandboxes/supervisor/src/index.test.ts index 84d1753..8bdb03d 100644 --- a/infra/sandboxes/supervisor/src/index.test.ts +++ b/infra/sandboxes/supervisor/src/index.test.ts @@ -203,10 +203,9 @@ describe("sandbox supervisor HTTP boundary", () => { expect(needsBridgeBrokerListener("0.0.0.0", "linux", false)).toBe(false); }); - it("shares the GitHub broker socket from any Linux host supervisor", () => { - expect(servesGithubBrokerSocket("linux", false)).toBe(true); - expect(servesGithubBrokerSocket("linux", true)).toBe(false); - expect(servesGithubBrokerSocket("darwin", false)).toBe(false); + it("shares the GitHub broker socket from any Linux supervisor, containerized or not", () => { + expect(servesGithubBrokerSocket("linux")).toBe(true); + expect(servesGithubBrokerSocket("darwin")).toBe(false); }); it("keeps the larger file-write allowance scoped to that exact POST route", () => { diff --git a/infra/sandboxes/supervisor/src/index.ts b/infra/sandboxes/supervisor/src/index.ts index 1986396..45f1c48 100644 --- a/infra/sandboxes/supervisor/src/index.ts +++ b/infra/sandboxes/supervisor/src/index.ts @@ -2,7 +2,6 @@ import { randomUUID } from "node:crypto"; import { chmodSync, existsSync, mkdirSync, rmSync } from "node:fs"; import { mkdir } from "node:fs/promises"; import http from "node:http"; -import os from "node:os"; import path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; @@ -158,18 +157,15 @@ export function needsBridgeBrokerListener( // run as long; Node's 5-minute default would cut them. const serverOptions = { requestTimeout: 2 * 60 * 60 * 1000 }; -// Docker binds host paths, so only a supervisor running directly on a Linux host can share one. -export function servesGithubBrokerSocket( - platform: NodeJS.Platform = process.platform, - inContainer = existsSync("/.dockerenv"), -): boolean { - return platform === "linux" && !inContainer; +// Docker Desktop cannot pass a host unix socket into a container, so only Linux shares one. +export function servesGithubBrokerSocket(platform: NodeJS.Platform = process.platform): boolean { + return platform === "linux"; } // A socket bind-mounted into each computer reaches the broker without crossing the host firewall. +// It lives in the data directory, which a containerized supervisor already shares with the host. const githubBrokerSocketDir = path.resolve( - process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ?? - path.join(os.homedir(), ".rakazo", "github-broker"), + process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ?? path.join(dataDir, "github-broker"), ); // Set once the socket listens, so computers only get the mount when there is something behind it. let githubBrokerSocketReady = false; @@ -191,6 +187,14 @@ function listenGithubBrokerSocket(logger: ReturnType) { return server; } +// Where Docker finds the socket directory on the host. A data volume has no host path to bind, +// so those computers keep reaching the broker over the network. +function githubBrokerSocketHostDir(runtimeInfo: Docker.ContainerInspectInfo | undefined) { + if (!githubBrokerSocketReady) return undefined; + const storage = computerHomeStorage(githubBrokerSocketDir, dataDir, runtimeInfo); + return storage.homeVolume ? undefined : storage.homePath; +} + async function bridgeGatewayAddress(): Promise { const network = await docker.getNetwork("bridge").inspect(); return network.IPAM?.Config?.find((entry: { Gateway?: string }) => entry.Gateway)?.Gateway; @@ -256,6 +260,7 @@ app.post("/computers", async (c) => { assertVolumeSubpathSupport((await docker.version()).ApiVersion); } const computerUser = runtimeInfo ? COMPUTER_USER : hostComputerUser(hostUid, hostGid); + const brokerSocketDir = githubBrokerSocketHostDir(runtimeInfo); const existing = await findBotContainer(body.botId, body.spaceId); if (existing) { const info = await existing.inspect(); @@ -270,7 +275,7 @@ app.post("/computers", async (c) => { info.Config.User === computerUser && controlPublishOk && (!storage.homeVolume || homeVolumeMatches(info.HostConfig.Mounts, storage.homeVolume)) && - (!githubBrokerSocketReady || githubBrokerSocketMounted(info.Mounts)) + (!brokerSocketDir || githubBrokerSocketMounted(info.Mounts)) ) { if (!info.State.Running) await existing.start(); return c.json({ @@ -333,7 +338,7 @@ app.post("/computers", async (c) => { networkMode, controlToken: randomUUID(), publishControlPort: controlViaLoopback, - ...(githubBrokerSocketReady ? { githubBrokerSocketDir } : {}), + ...(brokerSocketDir ? { githubBrokerSocketDir: brokerSocketDir } : {}), }), ); await container.start(); diff --git a/ops/evo-deploy/README.md b/ops/evo-deploy/README.md index 45d59f0..61c84f8 100644 --- a/ops/evo-deploy/README.md +++ b/ops/evo-deploy/README.md @@ -176,6 +176,8 @@ Do not commit host passwords, tokens, or SSH private keys. Configure restart tar | `EVO_API_HEALTH_URL` | `http://127.0.0.1:3100/health` | API health probe | | `EVO_WEB_HEALTH_URL` | `http://127.0.0.1:5173/` | Web health probe | | `EVO_HEALTH_TIMEOUT_SEC` | `120` | Per-check timeout (seconds) | +| `EVO_SUPERVISOR_CONTAINER` | `evoo-sandbox-supervisor` | Standalone supervisor container rebuilt and recreated with its settings when its sources change | +| `EVO_SUPERVISOR_HEALTH_URL` | `http://127.0.0.1:7091/health` | Supervisor health probe; an unhealthy new container is replaced by the previous one | `start-stack.sh`: diff --git a/ops/evo-deploy/deploy.sh b/ops/evo-deploy/deploy.sh index 023ef65..ba34854 100755 --- a/ops/evo-deploy/deploy.sh +++ b/ops/evo-deploy/deploy.sh @@ -480,6 +480,73 @@ WHERE "state" = 'stopped' SQL } +# A host that runs the sandbox supervisor as a standalone container keeps the image it was started +# with. Rebuild it when its sources changed and start a fresh container with the same name, image, +# network, restart policy, mounts and environment; the old one comes back if the new one is unhealthy. +refresh_supervisor_container() { + local previous_sha="$1" target_sha="$2" + local name="${EVO_SUPERVISOR_CONTAINER:-evoo-sandbox-supervisor}" + local health_url="${EVO_SUPERVISOR_HEALTH_URL:-http://127.0.0.1:7091/health}" + git -C "$EVO_DEPLOY_DIR" diff --quiet "$previous_sha" "$target_sha" -- \ + infra/sandboxes/supervisor packages/contracts packages/core packages/logging pnpm-lock.yaml && return 0 + command -v docker >/dev/null 2>&1 || return 0 + docker container inspect "$name" >/dev/null 2>&1 || return 0 + if [[ "$DRY_RUN" == true ]]; then + log "[dry-run] would rebuild and recreate the $name container" + return 0 + fi + + local work image network restart retries line + work="$(mktemp -d)" + image="$(docker container inspect -f '{{.Config.Image}}' "$name")" + network="$(docker container inspect -f '{{.HostConfig.NetworkMode}}' "$name")" + restart="$(docker container inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$name")" + retries="$(docker container inspect -f '{{.HostConfig.RestartPolicy.MaximumRetryCount}}' "$name")" + local -a args=() + [[ -n "$network" && "$network" != default && "$network" != bridge ]] && args+=(--network "$network") + if [[ "$restart" == on-failure && "$retries" != 0 ]]; then + args+=(--restart "on-failure:$retries") + elif [[ -n "$restart" && "$restart" != no ]]; then + args+=(--restart "$restart") + fi + while IFS= read -r line; do + [[ -n "$line" ]] && args+=(-v "$line") + done < <(docker container inspect -f \ + '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}{{else}}{{.Source}}{{end}}:{{.Destination}}{{if not .RW}}:ro{{end}}{{"\n"}}{{end}}' "$name") + while IFS= read -r line; do + [[ -n "$line" ]] && args+=(-p "${line#:}") + done < <(docker container inspect -f \ + '{{range $port, $bindings := .HostConfig.PortBindings}}{{range $bindings}}{{.HostIp}}:{{.HostPort}}:{{$port}}{{"\n"}}{{end}}{{end}}' "$name") + # The image sets PATH and its Node versions itself. + (umask 077 && docker container inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$name" | + grep -Ev '^(PATH|NODE_VERSION|YARN_VERSION)=' >"$work/env") || true + args+=(--env-file "$work/env") + + log "rebuilding the $name container" + if ! docker build -q -t "$image" -f "$EVO_DEPLOY_DIR/infra/sandboxes/supervisor/Dockerfile" "$EVO_DEPLOY_DIR" >&2; then + log "WARNING: the supervisor image rebuild failed; $name keeps running the old image" + rm -rf "$work" + return 0 + fi + docker rm -f "$name-previous" >/dev/null 2>&1 || true + if ! { docker stop "$name" >/dev/null && docker rename "$name" "$name-previous"; }; then + log "WARNING: could not stop $name; it keeps running the old image" + docker start "$name" >/dev/null 2>&1 || true + rm -rf "$work" + return 0 + fi + if docker run -d --name "$name" "${args[@]}" "$image" >/dev/null && wait_for_url "$health_url" "supervisor"; then + docker rm "$name-previous" >/dev/null + log "$name now runs the new image" + else + log "WARNING: the new $name container did not come up; restoring the previous one" + docker rm -f "$name" >/dev/null 2>&1 || true + { docker rename "$name-previous" "$name" && docker start "$name" >/dev/null; } || + log "WARNING: could not restore $name; start it with docker start $name-previous" + fi + rm -rf "$work" +} + main() { parse_args "$@" validate_deploy_dir @@ -509,6 +576,7 @@ main() { write_state "$previous_sha" "$target_sha" "success" rebuild_computer_image "$previous_sha" "$target_sha" + refresh_supervisor_container "$previous_sha" "$target_sha" recreate_idle_computers "$previous_sha" "$target_sha" log "deploy succeeded: $previous_sha -> $target_sha" } diff --git a/packages/testkit/src/evo-deploy.test.ts b/packages/testkit/src/evo-deploy.test.ts index 677cf4f..c886657 100644 --- a/packages/testkit/src/evo-deploy.test.ts +++ b/packages/testkit/src/evo-deploy.test.ts @@ -49,6 +49,7 @@ interface Revision { build?: "ok" | "fail"; health?: "ok" | "down"; computer?: string; + supervisor?: string; } /** @@ -122,6 +123,10 @@ printf 'restart %s %s\\n' "$(cat "$EVO_DEPLOY_DIR/version.txt")" "$served" >> "$ path.join(checkout, "infra/sandboxes/computer/Dockerfile"), revision.computer ?? "FROM a\n", ); + write( + path.join(checkout, "infra/sandboxes/supervisor/Dockerfile"), + revision.supervisor ?? "FROM a\n", + ); git("add", "-A"); git("commit", "-q", "-m", version); return git("rev-parse", "HEAD"); @@ -196,6 +201,53 @@ describe("evo deploy", () => { expect(cmds.at(-1)).toBe("B --filter @rakazo/db exec prisma db execute --stdin"); }); + it("recreates a supervisor container with its settings when its sources changed", () => { + const fakeDocker = (bin: string) => + write( + path.join(bin, "docker"), + `#!/bin/bash +if [[ "$1 $2" == "container inspect" ]]; then + case "$4" in + *Config.Image*) echo rakazo/sandbox-supervisor:local ;; + *NetworkMode*) echo host ;; + *RestartPolicy.Name*) echo unless-stopped ;; + *MaximumRetryCount*) echo 0 ;; + *Mounts*) printf '/srv/data:/srv/data\\n/var/run/docker.sock:/var/run/docker.sock\\n' ;; + *Config.Env*) printf 'DATA_DIR=/srv/data\\nPATH=/usr/bin\\nSUPERVISOR_PORT=7091\\n' ;; + esac + exit 0 +fi +if [[ "$1" == run ]]; then + for arg in "$@"; do [[ "$prev" == --env-file ]] && cp "$arg" "$(dirname "$COMMAND_LOG")/supervisor.env"; prev="$arg"; done +fi +echo "docker $*" | sed "s|--env-file [^ ]*|--env-file ENV|" >> "$COMMAND_LOG" +`, + 0o755, + ); + + const unchanged = deployFixture({}); + fakeDocker(path.join(unchanged.checkout, "../bin")); + expect(unchanged.run().status).toBe(0); + expect(unchanged.commands().some((line) => line.startsWith("docker build"))).toBe(false); + + const changed = deployFixture({ supervisor: "FROM b\n" }); + fakeDocker(path.join(changed.checkout, "../bin")); + const result = changed.run(); + expect(result.status, result.stderr).toBe(0); + const docker = changed.commands().filter((line) => line.startsWith("docker")); + expect(docker).toEqual([ + `docker build -q -t rakazo/sandbox-supervisor:local -f ${path.join(changed.checkout, "infra/sandboxes/supervisor/Dockerfile")} ${changed.checkout}`, + "docker rm -f evoo-sandbox-supervisor-previous", + "docker stop evoo-sandbox-supervisor", + "docker rename evoo-sandbox-supervisor evoo-sandbox-supervisor-previous", + "docker run -d --name evoo-sandbox-supervisor --network host --restart unless-stopped -v /srv/data:/srv/data -v /var/run/docker.sock:/var/run/docker.sock --env-file ENV rakazo/sandbox-supervisor:local", + "docker rm evoo-sandbox-supervisor-previous", + ]); + expect(readFileSync(path.join(changed.checkout, "../supervisor.env"), "utf8")).toBe( + "DATA_DIR=/srv/data\nSUPERVISOR_PORT=7091\n", + ); + }); + it("stops a failing build before the restart and keeps the served build", () => { const f = deployFixture({ build: "fail" }); const result = f.run();