diff --git a/infra/sandboxes/supervisor/src/index.test.ts b/infra/sandboxes/supervisor/src/index.test.ts index 84d1753..8bdb03d 100644 --- a/infra/sandboxes/supervisor/src/index.test.ts +++ b/infra/sandboxes/supervisor/src/index.test.ts @@ -203,10 +203,9 @@ describe("sandbox supervisor HTTP boundary", () => { expect(needsBridgeBrokerListener("0.0.0.0", "linux", false)).toBe(false); }); - it("shares the GitHub broker socket from any Linux host supervisor", () => { - expect(servesGithubBrokerSocket("linux", false)).toBe(true); - expect(servesGithubBrokerSocket("linux", true)).toBe(false); - expect(servesGithubBrokerSocket("darwin", false)).toBe(false); + it("shares the GitHub broker socket from any Linux supervisor, containerized or not", () => { + expect(servesGithubBrokerSocket("linux")).toBe(true); + expect(servesGithubBrokerSocket("darwin")).toBe(false); }); it("keeps the larger file-write allowance scoped to that exact POST route", () => { diff --git a/infra/sandboxes/supervisor/src/index.ts b/infra/sandboxes/supervisor/src/index.ts index 1986396..45f1c48 100644 --- a/infra/sandboxes/supervisor/src/index.ts +++ b/infra/sandboxes/supervisor/src/index.ts @@ -2,7 +2,6 @@ import { randomUUID } from "node:crypto"; import { chmodSync, existsSync, mkdirSync, rmSync } from "node:fs"; import { mkdir } from "node:fs/promises"; import http from "node:http"; -import os from "node:os"; import path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; @@ -158,18 +157,15 @@ export function needsBridgeBrokerListener( // run as long; Node's 5-minute default would cut them. const serverOptions = { requestTimeout: 2 * 60 * 60 * 1000 }; -// Docker binds host paths, so only a supervisor running directly on a Linux host can share one. -export function servesGithubBrokerSocket( - platform: NodeJS.Platform = process.platform, - inContainer = existsSync("/.dockerenv"), -): boolean { - return platform === "linux" && !inContainer; +// Docker Desktop cannot pass a host unix socket into a container, so only Linux shares one. +export function servesGithubBrokerSocket(platform: NodeJS.Platform = process.platform): boolean { + return platform === "linux"; } // A socket bind-mounted into each computer reaches the broker without crossing the host firewall. +// It lives in the data directory, which a containerized supervisor already shares with the host. const githubBrokerSocketDir = path.resolve( - process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ?? - path.join(os.homedir(), ".rakazo", "github-broker"), + process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ?? path.join(dataDir, "github-broker"), ); // Set once the socket listens, so computers only get the mount when there is something behind it. let githubBrokerSocketReady = false; @@ -191,6 +187,14 @@ function listenGithubBrokerSocket(logger: ReturnType) { return server; } +// Where Docker finds the socket directory on the host. A data volume has no host path to bind, +// so those computers keep reaching the broker over the network. +function githubBrokerSocketHostDir(runtimeInfo: Docker.ContainerInspectInfo | undefined) { + if (!githubBrokerSocketReady) return undefined; + const storage = computerHomeStorage(githubBrokerSocketDir, dataDir, runtimeInfo); + return storage.homeVolume ? undefined : storage.homePath; +} + async function bridgeGatewayAddress(): Promise { const network = await docker.getNetwork("bridge").inspect(); return network.IPAM?.Config?.find((entry: { Gateway?: string }) => entry.Gateway)?.Gateway; @@ -256,6 +260,7 @@ app.post("/computers", async (c) => { assertVolumeSubpathSupport((await docker.version()).ApiVersion); } const computerUser = runtimeInfo ? COMPUTER_USER : hostComputerUser(hostUid, hostGid); + const brokerSocketDir = githubBrokerSocketHostDir(runtimeInfo); const existing = await findBotContainer(body.botId, body.spaceId); if (existing) { const info = await existing.inspect(); @@ -270,7 +275,7 @@ app.post("/computers", async (c) => { info.Config.User === computerUser && controlPublishOk && (!storage.homeVolume || homeVolumeMatches(info.HostConfig.Mounts, storage.homeVolume)) && - (!githubBrokerSocketReady || githubBrokerSocketMounted(info.Mounts)) + (!brokerSocketDir || githubBrokerSocketMounted(info.Mounts)) ) { if (!info.State.Running) await existing.start(); return c.json({ @@ -333,7 +338,7 @@ app.post("/computers", async (c) => { networkMode, controlToken: randomUUID(), publishControlPort: controlViaLoopback, - ...(githubBrokerSocketReady ? { githubBrokerSocketDir } : {}), + ...(brokerSocketDir ? { githubBrokerSocketDir: brokerSocketDir } : {}), }), ); await container.start(); diff --git a/ops/evo-deploy/README.md b/ops/evo-deploy/README.md index 45d59f0..61c84f8 100644 --- a/ops/evo-deploy/README.md +++ b/ops/evo-deploy/README.md @@ -176,6 +176,8 @@ Do not commit host passwords, tokens, or SSH private keys. Configure restart tar | `EVO_API_HEALTH_URL` | `http://127.0.0.1:3100/health` | API health probe | | `EVO_WEB_HEALTH_URL` | `http://127.0.0.1:5173/` | Web health probe | | `EVO_HEALTH_TIMEOUT_SEC` | `120` | Per-check timeout (seconds) | +| `EVO_SUPERVISOR_CONTAINER` | `evoo-sandbox-supervisor` | Standalone supervisor container rebuilt and recreated with its settings when its sources change | +| `EVO_SUPERVISOR_HEALTH_URL` | `http://127.0.0.1:7091/health` | Supervisor health probe; an unhealthy new container is replaced by the previous one | `start-stack.sh`: diff --git a/ops/evo-deploy/deploy.sh b/ops/evo-deploy/deploy.sh index 023ef65..ba34854 100755 --- a/ops/evo-deploy/deploy.sh +++ b/ops/evo-deploy/deploy.sh @@ -480,6 +480,73 @@ WHERE "state" = 'stopped' SQL } +# A host that runs the sandbox supervisor as a standalone container keeps the image it was started +# with. Rebuild it when its sources changed and start a fresh container with the same name, image, +# network, restart policy, mounts and environment; the old one comes back if the new one is unhealthy. +refresh_supervisor_container() { + local previous_sha="$1" target_sha="$2" + local name="${EVO_SUPERVISOR_CONTAINER:-evoo-sandbox-supervisor}" + local health_url="${EVO_SUPERVISOR_HEALTH_URL:-http://127.0.0.1:7091/health}" + git -C "$EVO_DEPLOY_DIR" diff --quiet "$previous_sha" "$target_sha" -- \ + infra/sandboxes/supervisor packages/contracts packages/core packages/logging pnpm-lock.yaml && return 0 + command -v docker >/dev/null 2>&1 || return 0 + docker container inspect "$name" >/dev/null 2>&1 || return 0 + if [[ "$DRY_RUN" == true ]]; then + log "[dry-run] would rebuild and recreate the $name container" + return 0 + fi + + local work image network restart retries line + work="$(mktemp -d)" + image="$(docker container inspect -f '{{.Config.Image}}' "$name")" + network="$(docker container inspect -f '{{.HostConfig.NetworkMode}}' "$name")" + restart="$(docker container inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$name")" + retries="$(docker container inspect -f '{{.HostConfig.RestartPolicy.MaximumRetryCount}}' "$name")" + local -a args=() + [[ -n "$network" && "$network" != default && "$network" != bridge ]] && args+=(--network "$network") + if [[ "$restart" == on-failure && "$retries" != 0 ]]; then + args+=(--restart "on-failure:$retries") + elif [[ -n "$restart" && "$restart" != no ]]; then + args+=(--restart "$restart") + fi + while IFS= read -r line; do + [[ -n "$line" ]] && args+=(-v "$line") + done < <(docker container inspect -f \ + '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}{{else}}{{.Source}}{{end}}:{{.Destination}}{{if not .RW}}:ro{{end}}{{"\n"}}{{end}}' "$name") + while IFS= read -r line; do + [[ -n "$line" ]] && args+=(-p "${line#:}") + done < <(docker container inspect -f \ + '{{range $port, $bindings := .HostConfig.PortBindings}}{{range $bindings}}{{.HostIp}}:{{.HostPort}}:{{$port}}{{"\n"}}{{end}}{{end}}' "$name") + # The image sets PATH and its Node versions itself. + (umask 077 && docker container inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$name" | + grep -Ev '^(PATH|NODE_VERSION|YARN_VERSION)=' >"$work/env") || true + args+=(--env-file "$work/env") + + log "rebuilding the $name container" + if ! docker build -q -t "$image" -f "$EVO_DEPLOY_DIR/infra/sandboxes/supervisor/Dockerfile" "$EVO_DEPLOY_DIR" >&2; then + log "WARNING: the supervisor image rebuild failed; $name keeps running the old image" + rm -rf "$work" + return 0 + fi + docker rm -f "$name-previous" >/dev/null 2>&1 || true + if ! { docker stop "$name" >/dev/null && docker rename "$name" "$name-previous"; }; then + log "WARNING: could not stop $name; it keeps running the old image" + docker start "$name" >/dev/null 2>&1 || true + rm -rf "$work" + return 0 + fi + if docker run -d --name "$name" "${args[@]}" "$image" >/dev/null && wait_for_url "$health_url" "supervisor"; then + docker rm "$name-previous" >/dev/null + log "$name now runs the new image" + else + log "WARNING: the new $name container did not come up; restoring the previous one" + docker rm -f "$name" >/dev/null 2>&1 || true + { docker rename "$name-previous" "$name" && docker start "$name" >/dev/null; } || + log "WARNING: could not restore $name; start it with docker start $name-previous" + fi + rm -rf "$work" +} + main() { parse_args "$@" validate_deploy_dir @@ -509,6 +576,7 @@ main() { write_state "$previous_sha" "$target_sha" "success" rebuild_computer_image "$previous_sha" "$target_sha" + refresh_supervisor_container "$previous_sha" "$target_sha" recreate_idle_computers "$previous_sha" "$target_sha" log "deploy succeeded: $previous_sha -> $target_sha" } diff --git a/packages/testkit/src/evo-deploy.test.ts b/packages/testkit/src/evo-deploy.test.ts index 677cf4f..c886657 100644 --- a/packages/testkit/src/evo-deploy.test.ts +++ b/packages/testkit/src/evo-deploy.test.ts @@ -49,6 +49,7 @@ interface Revision { build?: "ok" | "fail"; health?: "ok" | "down"; computer?: string; + supervisor?: string; } /** @@ -122,6 +123,10 @@ printf 'restart %s %s\\n' "$(cat "$EVO_DEPLOY_DIR/version.txt")" "$served" >> "$ path.join(checkout, "infra/sandboxes/computer/Dockerfile"), revision.computer ?? "FROM a\n", ); + write( + path.join(checkout, "infra/sandboxes/supervisor/Dockerfile"), + revision.supervisor ?? "FROM a\n", + ); git("add", "-A"); git("commit", "-q", "-m", version); return git("rev-parse", "HEAD"); @@ -196,6 +201,53 @@ describe("evo deploy", () => { expect(cmds.at(-1)).toBe("B --filter @rakazo/db exec prisma db execute --stdin"); }); + it("recreates a supervisor container with its settings when its sources changed", () => { + const fakeDocker = (bin: string) => + write( + path.join(bin, "docker"), + `#!/bin/bash +if [[ "$1 $2" == "container inspect" ]]; then + case "$4" in + *Config.Image*) echo rakazo/sandbox-supervisor:local ;; + *NetworkMode*) echo host ;; + *RestartPolicy.Name*) echo unless-stopped ;; + *MaximumRetryCount*) echo 0 ;; + *Mounts*) printf '/srv/data:/srv/data\\n/var/run/docker.sock:/var/run/docker.sock\\n' ;; + *Config.Env*) printf 'DATA_DIR=/srv/data\\nPATH=/usr/bin\\nSUPERVISOR_PORT=7091\\n' ;; + esac + exit 0 +fi +if [[ "$1" == run ]]; then + for arg in "$@"; do [[ "$prev" == --env-file ]] && cp "$arg" "$(dirname "$COMMAND_LOG")/supervisor.env"; prev="$arg"; done +fi +echo "docker $*" | sed "s|--env-file [^ ]*|--env-file ENV|" >> "$COMMAND_LOG" +`, + 0o755, + ); + + const unchanged = deployFixture({}); + fakeDocker(path.join(unchanged.checkout, "../bin")); + expect(unchanged.run().status).toBe(0); + expect(unchanged.commands().some((line) => line.startsWith("docker build"))).toBe(false); + + const changed = deployFixture({ supervisor: "FROM b\n" }); + fakeDocker(path.join(changed.checkout, "../bin")); + const result = changed.run(); + expect(result.status, result.stderr).toBe(0); + const docker = changed.commands().filter((line) => line.startsWith("docker")); + expect(docker).toEqual([ + `docker build -q -t rakazo/sandbox-supervisor:local -f ${path.join(changed.checkout, "infra/sandboxes/supervisor/Dockerfile")} ${changed.checkout}`, + "docker rm -f evoo-sandbox-supervisor-previous", + "docker stop evoo-sandbox-supervisor", + "docker rename evoo-sandbox-supervisor evoo-sandbox-supervisor-previous", + "docker run -d --name evoo-sandbox-supervisor --network host --restart unless-stopped -v /srv/data:/srv/data -v /var/run/docker.sock:/var/run/docker.sock --env-file ENV rakazo/sandbox-supervisor:local", + "docker rm evoo-sandbox-supervisor-previous", + ]); + expect(readFileSync(path.join(changed.checkout, "../supervisor.env"), "utf8")).toBe( + "DATA_DIR=/srv/data\nSUPERVISOR_PORT=7091\n", + ); + }); + it("stops a failing build before the restart and keeps the served build", () => { const f = deployFixture({ build: "fail" }); const result = f.run();