Give every computer the GitHub broker socket and name failed paths (#214)
Chief's computer still timed out reaching the GitHub broker after the socket mount shipped. A reused computer kept its old container whenever the image matched, so one created without the mount never got it, and the socket was only served for a loopback-bound supervisor. - Recreate a reused computer that lacks the socket mount once the supervisor serves the socket. - Serve the socket from any supervisor running directly on a Linux host. - Give the broker listeners the same long request timeout as the main server, so large brokered clones are not cut after five minutes. - The broker client now says which path failed (no socket, socket exit, network exit) so a relayed error points at the cause. Claude-Session: https://claude.ai/code/session_01Tgvrj34NMnaqvtgvruYQZZ Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
parent
7452b8c6d2
commit
9bc3187a4a
@ -25,14 +25,19 @@ for line in open("/proc/net/route").readlines()[1:]:
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
' "$RAKAZO_GITHUB_BROKER_URL"
|
' "$RAKAZO_GITHUB_BROKER_URL"
|
||||||
}
|
}
|
||||||
|
# Names each path that failed so the user can pass the exact cause on.
|
||||||
unreachable() {
|
unreachable() {
|
||||||
echo "evOo could not reach its GitHub connection from this computer (curl exit $status). Tell the user GitHub access is broken on the evOo server; do not ask for a token." >&2
|
echo "evOo could not reach its GitHub connection from this computer (${attempts}curl exit $status). Tell the user GitHub access is broken on the evOo server and quote this line; do not ask for a token." >&2
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
status=0
|
status=0
|
||||||
|
attempts=""
|
||||||
socket=/run/rakazo-github-broker/broker.sock
|
socket=/run/rakazo-github-broker/broker.sock
|
||||||
if [[ -S "$socket" ]]; then
|
if [[ -S "$socket" ]]; then
|
||||||
response="$(post http://broker/internal/github-broker --unix-socket "$socket" 2>/dev/null)" || status=$?
|
response="$(post http://broker/internal/github-broker --unix-socket "$socket" 2>/dev/null)" || status=$?
|
||||||
|
[[ "$status" == 0 ]] || attempts="socket exit $status; "
|
||||||
|
else
|
||||||
|
attempts="no socket; "
|
||||||
fi
|
fi
|
||||||
# 7: nothing listening on the socket either; then try the network.
|
# 7: nothing listening on the socket either; then try the network.
|
||||||
if [[ ! -S "$socket" || "$status" == 7 ]]; then
|
if [[ ! -S "$socket" || "$status" == 7 ]]; then
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
import { execFileSync } from "node:child_process";
|
import { execFileSync, spawnSync } from "node:child_process";
|
||||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
@ -52,4 +52,18 @@ describe("computer git and gh shims", () => {
|
|||||||
expect(endpoint).toBe("v1/gh");
|
expect(endpoint).toBe("v1/gh");
|
||||||
expect(body.argv).toEqual(["gh", "pr", "list"]);
|
expect(body.argv).toEqual(["gh", "pr", "list"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("names the failed path when the broker is unreachable", () => {
|
||||||
|
// Port 1 refuses at once, and a plain loopback URL has no gateway fallback.
|
||||||
|
const result = spawnSync("bash", [shimPath("rakazo-broker"), "v1/git", "{}"], {
|
||||||
|
encoding: "utf8",
|
||||||
|
env: {
|
||||||
|
PATH: process.env.PATH,
|
||||||
|
RAKAZO_GITHUB_BROKER_URL: "http://127.0.0.1:1/internal/github-broker",
|
||||||
|
RAKAZO_GITHUB_BROKER_RUN_TOKEN: "rgb1.test",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(result.status).toBe(1);
|
||||||
|
expect(result.stderr).toContain("(no socket; curl exit 7)");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -21,6 +21,7 @@ import {
|
|||||||
containerCreateOptions,
|
containerCreateOptions,
|
||||||
containerNameFor,
|
containerNameFor,
|
||||||
controlPortPublicationMatches,
|
controlPortPublicationMatches,
|
||||||
|
githubBrokerSocketMounted,
|
||||||
homeVolumeMatches,
|
homeVolumeMatches,
|
||||||
hostComputerUser,
|
hostComputerUser,
|
||||||
legacyNetworkOwnedSolelyBy,
|
legacyNetworkOwnedSolelyBy,
|
||||||
@ -131,6 +132,16 @@ describe("graphical computer spec", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("recognizes computers that already carry the GitHub broker socket", () => {
|
||||||
|
const mount = (Destination: string) =>
|
||||||
|
({ Destination }) as Docker.ContainerInspectInfo["Mounts"][number];
|
||||||
|
expect(
|
||||||
|
githubBrokerSocketMounted([mount("/home/rakazo"), mount("/run/rakazo-github-broker")]),
|
||||||
|
).toBe(true);
|
||||||
|
expect(githubBrokerSocketMounted([mount("/home/rakazo")])).toBe(false);
|
||||||
|
expect(githubBrokerSocketMounted(undefined)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("still publishes host ports when NetworkMode is a per-bot isolated network", () => {
|
it("still publishes host ports when NetworkMode is a per-bot isolated network", () => {
|
||||||
const networkMode = computerNetworkNameFor("bot_isolation");
|
const networkMode = computerNetworkNameFor("bot_isolation");
|
||||||
const options = containerCreateOptions({
|
const options = containerCreateOptions({
|
||||||
|
|||||||
@ -252,6 +252,13 @@ export interface ComputerCreateInput {
|
|||||||
/** Where computers find the GitHub broker socket; the git shim prefers it over the network. */
|
/** Where computers find the GitHub broker socket; the git shim prefers it over the network. */
|
||||||
export const GITHUB_BROKER_SOCKET_MOUNT = "/run/rakazo-github-broker";
|
export const GITHUB_BROKER_SOCKET_MOUNT = "/run/rakazo-github-broker";
|
||||||
|
|
||||||
|
/** Computers created before the socket existed lack the mount and must be recreated to get it. */
|
||||||
|
export function githubBrokerSocketMounted(
|
||||||
|
mounts: Docker.ContainerInspectInfo["Mounts"] | undefined,
|
||||||
|
) {
|
||||||
|
return mounts?.some((mount) => mount.Destination === GITHUB_BROKER_SOCKET_MOUNT) ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
interface PointerInput {
|
interface PointerInput {
|
||||||
kind: "pointer";
|
kind: "pointer";
|
||||||
x: number;
|
x: number;
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import {
|
|||||||
MAX_SUPERVISOR_REQUEST_BYTES,
|
MAX_SUPERVISOR_REQUEST_BYTES,
|
||||||
needsBridgeBrokerListener,
|
needsBridgeBrokerListener,
|
||||||
resolveDockerSocketPath,
|
resolveDockerSocketPath,
|
||||||
|
servesGithubBrokerSocket,
|
||||||
supervisorApp,
|
supervisorApp,
|
||||||
supervisorRequestBodyLimit,
|
supervisorRequestBodyLimit,
|
||||||
waitForScreenReady,
|
waitForScreenReady,
|
||||||
@ -202,6 +203,12 @@ describe("sandbox supervisor HTTP boundary", () => {
|
|||||||
expect(needsBridgeBrokerListener("0.0.0.0", "linux", false)).toBe(false);
|
expect(needsBridgeBrokerListener("0.0.0.0", "linux", false)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("shares the GitHub broker socket from any Linux host supervisor", () => {
|
||||||
|
expect(servesGithubBrokerSocket("linux", false)).toBe(true);
|
||||||
|
expect(servesGithubBrokerSocket("linux", true)).toBe(false);
|
||||||
|
expect(servesGithubBrokerSocket("darwin", false)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps the larger file-write allowance scoped to that exact POST route", () => {
|
it("keeps the larger file-write allowance scoped to that exact POST route", () => {
|
||||||
expect(supervisorRequestBodyLimit("POST", "/computers/id/files")).toBe(
|
expect(supervisorRequestBodyLimit("POST", "/computers/id/files")).toBe(
|
||||||
MAX_SUPERVISOR_FILE_REQUEST_BYTES,
|
MAX_SUPERVISOR_FILE_REQUEST_BYTES,
|
||||||
|
|||||||
@ -36,6 +36,7 @@ import {
|
|||||||
containerCreateOptions,
|
containerCreateOptions,
|
||||||
containerNameFor,
|
containerNameFor,
|
||||||
controlPortPublicationMatches,
|
controlPortPublicationMatches,
|
||||||
|
githubBrokerSocketMounted,
|
||||||
homeVolumeMatches,
|
homeVolumeMatches,
|
||||||
hostComputerUser,
|
hostComputerUser,
|
||||||
legacyNetworkOwnedSolelyBy,
|
legacyNetworkOwnedSolelyBy,
|
||||||
@ -153,6 +154,18 @@ export function needsBridgeBrokerListener(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Streamed file writes copy whole repository archives, and brokered clones of large repositories
|
||||||
|
// run as long; Node's 5-minute default would cut them.
|
||||||
|
const serverOptions = { requestTimeout: 2 * 60 * 60 * 1000 };
|
||||||
|
|
||||||
|
// Docker binds host paths, so only a supervisor running directly on a Linux host can share one.
|
||||||
|
export function servesGithubBrokerSocket(
|
||||||
|
platform: NodeJS.Platform = process.platform,
|
||||||
|
inContainer = existsSync("/.dockerenv"),
|
||||||
|
): boolean {
|
||||||
|
return platform === "linux" && !inContainer;
|
||||||
|
}
|
||||||
|
|
||||||
// A socket bind-mounted into each computer reaches the broker without crossing the host firewall.
|
// A socket bind-mounted into each computer reaches the broker without crossing the host firewall.
|
||||||
const githubBrokerSocketDir = path.resolve(
|
const githubBrokerSocketDir = path.resolve(
|
||||||
process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ??
|
process.env.SANDBOX_GITHUB_BROKER_SOCKET_DIR ??
|
||||||
@ -165,7 +178,7 @@ function listenGithubBrokerSocket(logger: ReturnType<typeof createRootLogger>) {
|
|||||||
mkdirSync(githubBrokerSocketDir, { recursive: true, mode: 0o755 });
|
mkdirSync(githubBrokerSocketDir, { recursive: true, mode: 0o755 });
|
||||||
const socketPath = path.join(githubBrokerSocketDir, "broker.sock");
|
const socketPath = path.join(githubBrokerSocketDir, "broker.sock");
|
||||||
rmSync(socketPath, { force: true });
|
rmSync(socketPath, { force: true });
|
||||||
const server = createAdaptorServer({ fetch: brokerApp.fetch });
|
const server = createAdaptorServer({ fetch: brokerApp.fetch, serverOptions });
|
||||||
server.on("error", (error) => {
|
server.on("error", (error) => {
|
||||||
logger.warn("github broker socket unavailable", { "error.message": error.message });
|
logger.warn("github broker socket unavailable", { "error.message": error.message });
|
||||||
});
|
});
|
||||||
@ -256,7 +269,8 @@ app.post("/computers", async (c) => {
|
|||||||
(!networkMode || info.HostConfig.NetworkMode === networkMode) &&
|
(!networkMode || info.HostConfig.NetworkMode === networkMode) &&
|
||||||
info.Config.User === computerUser &&
|
info.Config.User === computerUser &&
|
||||||
controlPublishOk &&
|
controlPublishOk &&
|
||||||
(!storage.homeVolume || homeVolumeMatches(info.HostConfig.Mounts, storage.homeVolume))
|
(!storage.homeVolume || homeVolumeMatches(info.HostConfig.Mounts, storage.homeVolume)) &&
|
||||||
|
(!githubBrokerSocketReady || githubBrokerSocketMounted(info.Mounts))
|
||||||
) {
|
) {
|
||||||
if (!info.State.Running) await existing.start();
|
if (!info.State.Running) await existing.start();
|
||||||
return c.json({
|
return c.json({
|
||||||
@ -896,22 +910,24 @@ function startSupervisor() {
|
|||||||
computerResourceLimits();
|
computerResourceLimits();
|
||||||
const port = Number(process.env.SUPERVISOR_PORT ?? 7091);
|
const port = Number(process.env.SUPERVISOR_PORT ?? 7091);
|
||||||
const hostname = process.env.SUPERVISOR_HOST ?? "127.0.0.1";
|
const hostname = process.env.SUPERVISOR_HOST ?? "127.0.0.1";
|
||||||
// Streamed file writes copy whole repository archives; Node's 5-minute default would cut them.
|
|
||||||
const serverOptions = { requestTimeout: 2 * 60 * 60 * 1000 };
|
|
||||||
const server = serve({ fetch: app.fetch, hostname, port, serverOptions }, () => {
|
const server = serve({ fetch: app.fetch, hostname, port, serverOptions }, () => {
|
||||||
logger.info("supervisor listening", { "http.host": hostname, "http.port": port });
|
logger.info("supervisor listening", { "http.host": hostname, "http.port": port });
|
||||||
});
|
});
|
||||||
let brokerServer: ReturnType<typeof serve> | undefined;
|
let brokerServer: ReturnType<typeof serve> | undefined;
|
||||||
const brokerSocketServer = needsBridgeBrokerListener(hostname)
|
// The socket helps whatever the bind address: a host firewall can drop bridge traffic to 0.0.0.0 too.
|
||||||
|
const brokerSocketServer = servesGithubBrokerSocket()
|
||||||
? listenGithubBrokerSocket(logger)
|
? listenGithubBrokerSocket(logger)
|
||||||
: undefined;
|
: undefined;
|
||||||
if (needsBridgeBrokerListener(hostname)) {
|
if (needsBridgeBrokerListener(hostname)) {
|
||||||
void bridgeGatewayAddress()
|
void bridgeGatewayAddress()
|
||||||
.then((gateway) => {
|
.then((gateway) => {
|
||||||
if (!gateway) throw new Error("the default bridge network has no gateway");
|
if (!gateway) throw new Error("the default bridge network has no gateway");
|
||||||
brokerServer = serve({ fetch: brokerApp.fetch, hostname: gateway, port }, () => {
|
brokerServer = serve(
|
||||||
|
{ fetch: brokerApp.fetch, hostname: gateway, port, serverOptions },
|
||||||
|
() => {
|
||||||
logger.info("github broker listening", { "http.host": gateway, "http.port": port });
|
logger.info("github broker listening", { "http.host": gateway, "http.port": port });
|
||||||
});
|
},
|
||||||
|
);
|
||||||
})
|
})
|
||||||
.catch((error: unknown) => {
|
.catch((error: unknown) => {
|
||||||
logger.warn("github broker unavailable to computers", {
|
logger.warn("github broker unavailable to computers", {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user