Give every computer the GitHub broker socket and name failed paths (#214)
Chief's computer still timed out reaching the GitHub broker after the socket mount shipped. A reused computer kept its old container whenever the image matched, so one created without the mount never got it, and the socket was only served for a loopback-bound supervisor. - Recreate a reused computer that lacks the socket mount once the supervisor serves the socket. - Serve the socket from any supervisor running directly on a Linux host. - Give the broker listeners the same long request timeout as the main server, so large brokered clones are not cut after five minutes. - The broker client now says which path failed (no socket, socket exit, network exit) so a relayed error points at the cause. Claude-Session: https://claude.ai/code/session_01Tgvrj34NMnaqvtgvruYQZZ Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
parent
7452b8c6d2
commit
9bc3187a4a
@ -25,14 +25,19 @@ for line in open("/proc/net/route").readlines()[1:]:
|
||||
sys.exit(1)
|
||||
' "$RAKAZO_GITHUB_BROKER_URL"
|
||||
}
|
||||
# Names each path that failed so the user can pass the exact cause on.
|
||||
unreachable() {
|
||||
echo "evOo could not reach its GitHub connection from this computer (curl exit $status). Tell the user GitHub access is broken on the evOo server; do not ask for a token." >&2
|
||||
echo "evOo could not reach its GitHub connection from this computer (${attempts}curl exit $status). Tell the user GitHub access is broken on the evOo server and quote this line; do not ask for a token." >&2
|
||||
exit 1
|
||||
}
|
||||
status=0
|
||||
attempts=""
|
||||
socket=/run/rakazo-github-broker/broker.sock
|
||||
if [[ -S "$socket" ]]; then
|
||||
response="$(post http://broker/internal/github-broker --unix-socket "$socket" 2>/dev/null)" || status=$?
|
||||
[[ "$status" == 0 ]] || attempts="socket exit $status; "
|
||||
else
|
||||
attempts="no socket; "
|
||||
fi
|
||||
# 7: nothing listening on the socket either; then try the network.
|
||||
if [[ ! -S "$socket" || "$status" == 7 ]]; then
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
@ -52,4 +52,18 @@ describe("computer git and gh shims", () => {
|
||||
expect(endpoint).toBe("v1/gh");
|
||||
expect(body.argv).toEqual(["gh", "pr", "list"]);
|
||||
});
|
||||
|
||||
it("names the failed path when the broker is unreachable", () => {
|
||||
// Port 1 refuses at once, and a plain loopback URL has no gateway fallback.
|
||||
const result = spawnSync("bash", [shimPath("rakazo-broker"), "v1/git", "{}"], {
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
RAKAZO_GITHUB_BROKER_URL: "http://127.0.0.1:1/internal/github-broker",
|
||||
RAKAZO_GITHUB_BROKER_RUN_TOKEN: "rgb1.test",
|
||||
},
|
||||
});
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain("(no socket; curl exit 7)");
|
||||
});
|
||||
});
|
||||
|
||||
@ -21,6 +21,7 @@ import {
|
||||
containerCreateOptions,
|
||||
containerNameFor,
|
||||
controlPortPublicationMatches,
|
||||
githubBrokerSocketMounted,
|
||||
homeVolumeMatches,
|
||||
hostComputerUser,
|
||||
legacyNetworkOwnedSolelyBy,
|
||||
@ -131,6 +132,16 @@ describe("graphical computer spec", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("recognizes computers that already carry the GitHub broker socket", () => {
|
||||
const mount = (Destination: string) =>
|
||||
({ Destination }) as Docker.ContainerInspectInfo["Mounts"][number];
|
||||
expect(
|
||||
githubBrokerSocketMounted([mount("/home/rakazo"), mount("/run/rakazo-github-broker")]),
|
||||
).toBe(true);
|
||||
expect(githubBrokerSocketMounted([mount("/home/rakazo")])).toBe(false);
|
||||
expect(githubBrokerSocketMounted(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("still publishes host ports when NetworkMode is a per-bot isolated network", () => {
|
||||
const networkMode = computerNetworkNameFor("bot_isolation");
|
||||
const options = containerCreateOptions({
|
||||
|
||||
@ -252,6 +252,13 @@ export interface ComputerCreateInput {
|
||||
/** Where computers find the GitHub broker socket; the git shim prefers it over the network. */
|
||||
export const GITHUB_BROKER_SOCKET_MOUNT = "/run/rakazo-github-broker";
|
||||
|
||||
/** Computers created before the socket existed lack the mount and must be recreated to get it. */
|
||||
export function githubBrokerSocketMounted(
|
||||
mounts: Docker.ContainerInspectInfo["Mounts"] | undefined,
|
||||
) {
|
||||
return mounts?.some((mount) => mount.Destination === GITHUB_BROKER_SOCKET_MOUNT) ?? false;
|
||||
}
|
||||
|
||||
interface PointerInput {
|
||||
kind: "pointer";
|
||||
x: number;
|
||||
|
||||
@ -8,6 +8,7 @@ import {
|
||||
MAX_SUPERVISOR_REQUEST_BYTES,
|
||||
needsBridgeBrokerListener,
|
||||
resolveDockerSocketPath,
|
||||
servesGithubBrokerSocket,
|
||||
supervisorApp,
|
||||
supervisorRequestBodyLimit,
|
||||
waitForScreenReady,
|
||||
@ -202,6 +203,12 @@ describe("sandbox supervisor HTTP boundary", () => {
|
||||
expect(needsBridgeBrokerListener("0.0.0.0", "linux", false)).toBe(false);
|
||||
});
|
||||
|
||||
it("shares the GitHub broker socket from any Linux host supervisor", () => {
|
||||
expect(servesGithubBrokerSocket("linux", false)).toBe(true);
|
||||
expect(servesGithubBrokerSocket("linux", true)).toBe(false);
|
||||
expect(servesGithubBrokerSocket("darwin", false)).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps the larger file-write allowance scoped to that exact POST route", () => {
|
||||
expect(supervisorRequestBodyLimit("POST", "/computers/id/files")).toBe(
|
||||
MAX_SUPERVISOR_FILE_REQUEST_BYTES,
|
||||
|
||||
@ -36,6 +36,7 @@ import {
|
||||
containerCreateOptions,
|
||||
containerNameFor,
|
||||
controlPortPublicationMatches,
|
||||
githubBrokerSocketMounted,
|
||||
homeVolumeMatches,
|
||||
hostComputerUser,
|
||||
legacyNetworkOwnedSolelyBy,
|
||||
@ -153,6 +154,18 @@ export function needsBridgeBrokerListener(
|
||||
);
|
||||
}
|
||||
|
||||
// Streamed file writes copy whole repository archives, and brokered clones of large repositories
|
||||
// run as long; Node's 5-minute default would cut them.
|
||||
const serverOptions = { requestTimeout: 2 * 60 * 60 * 1000 };
|
||||
|
||||
// Docker binds host paths, so only a supervisor running directly on a Linux host can share one.
|
||||
export function servesGithubBrokerSocket(
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
inContainer = existsSync("/.dockerenv"),
|
||||
): boolean {
|
||||
return platform === "linux" && !inContainer;
|
||||
}
|
||||
|
||||
// A socket bind-mounted into each computer reaches the broker without crossing the host firewall.
|
||||
const githubBrokerSocketDir = path.resolve(
|
||||
process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ??
|
||||
@ -165,7 +178,7 @@ function listenGithubBrokerSocket(logger: ReturnType<typeof createRootLogger>) {
|
||||
mkdirSync(githubBrokerSocketDir, { recursive: true, mode: 0o755 });
|
||||
const socketPath = path.join(githubBrokerSocketDir, "broker.sock");
|
||||
rmSync(socketPath, { force: true });
|
||||
const server = createAdaptorServer({ fetch: brokerApp.fetch });
|
||||
const server = createAdaptorServer({ fetch: brokerApp.fetch, serverOptions });
|
||||
server.on("error", (error) => {
|
||||
logger.warn("github broker socket unavailable", { "error.message": error.message });
|
||||
});
|
||||
@ -256,7 +269,8 @@ app.post("/computers", async (c) => {
|
||||
(!networkMode || info.HostConfig.NetworkMode === networkMode) &&
|
||||
info.Config.User === computerUser &&
|
||||
controlPublishOk &&
|
||||
(!storage.homeVolume || homeVolumeMatches(info.HostConfig.Mounts, storage.homeVolume))
|
||||
(!storage.homeVolume || homeVolumeMatches(info.HostConfig.Mounts, storage.homeVolume)) &&
|
||||
(!githubBrokerSocketReady || githubBrokerSocketMounted(info.Mounts))
|
||||
) {
|
||||
if (!info.State.Running) await existing.start();
|
||||
return c.json({
|
||||
@ -896,22 +910,24 @@ function startSupervisor() {
|
||||
computerResourceLimits();
|
||||
const port = Number(process.env.SUPERVISOR_PORT ?? 7091);
|
||||
const hostname = process.env.SUPERVISOR_HOST ?? "127.0.0.1";
|
||||
// Streamed file writes copy whole repository archives; Node's 5-minute default would cut them.
|
||||
const serverOptions = { requestTimeout: 2 * 60 * 60 * 1000 };
|
||||
const server = serve({ fetch: app.fetch, hostname, port, serverOptions }, () => {
|
||||
logger.info("supervisor listening", { "http.host": hostname, "http.port": port });
|
||||
});
|
||||
let brokerServer: ReturnType<typeof serve> | undefined;
|
||||
const brokerSocketServer = needsBridgeBrokerListener(hostname)
|
||||
// The socket helps whatever the bind address: a host firewall can drop bridge traffic to 0.0.0.0 too.
|
||||
const brokerSocketServer = servesGithubBrokerSocket()
|
||||
? listenGithubBrokerSocket(logger)
|
||||
: undefined;
|
||||
if (needsBridgeBrokerListener(hostname)) {
|
||||
void bridgeGatewayAddress()
|
||||
.then((gateway) => {
|
||||
if (!gateway) throw new Error("the default bridge network has no gateway");
|
||||
brokerServer = serve({ fetch: brokerApp.fetch, hostname: gateway, port }, () => {
|
||||
brokerServer = serve(
|
||||
{ fetch: brokerApp.fetch, hostname: gateway, port, serverOptions },
|
||||
() => {
|
||||
logger.info("github broker listening", { "http.host": gateway, "http.port": port });
|
||||
});
|
||||
},
|
||||
);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
logger.warn("github broker unavailable to computers", {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user