diff --git a/apps/api/src/artifact-http.test.ts b/apps/api/src/artifact-http.test.ts index d95711f..e72784c 100644 --- a/apps/api/src/artifact-http.test.ts +++ b/apps/api/src/artifact-http.test.ts @@ -175,7 +175,7 @@ describe("upload timeouts", () => { it("gives the server time for large uploads but keeps the header timeout", () => { const server = createServer(API_SERVER_OPTIONS); - expect(server.requestTimeout).toBe(30 * 60 * 1000); + expect(server.requestTimeout).toBe(2 * 60 * 60 * 1000); expect(server.headersTimeout).toBe(createServer().headersTimeout); }); diff --git a/apps/api/src/artifact-http.ts b/apps/api/src/artifact-http.ts index d7485fd..0b4d8da 100644 --- a/apps/api/src/artifact-http.ts +++ b/apps/api/src/artifact-http.ts @@ -1,8 +1,8 @@ import type { AdapterContext, ArtifactByteRange, ArtifactStore } from "@rakazo/adapter-kit"; import { ArtifactTooLargeError } from "@rakazo/adapter-kit"; -import { artifactMaxBytes, storeArtifactStream } from "@rakazo/adapters"; +import { storeArtifactStream } from "@rakazo/adapters"; import type { Actor } from "@rakazo/contracts"; -import { attachmentKindOf, isAllowedAttachmentMimeType } from "@rakazo/contracts"; +import { artifactMaxBytes, attachmentKindOf, isAllowedAttachmentMimeType } from "@rakazo/contracts"; import type { PrismaClient } from "@rakazo/db"; import { createGroupRepos, createRepos, IsolationError, requireMembership } from "@rakazo/db"; import type { Context, Hono } from "hono"; @@ -11,11 +11,11 @@ import { cancelBody } from "./http-body.js"; export const ARTIFACT_HTTP_PATH = "/api/artifacts"; /** - * Node's default 5-minute requestTimeout cuts off a 500 MiB video upload on any uplink slower than - * about 14 Mbit/s; 30 minutes fits it at about 2.5 Mbit/s. headersTimeout keeps its default, and - * the per-upload stall timeout stops a slow-drip client from holding a connection that long. + * Node's default 5-minute requestTimeout cuts off a large upload on a slow uplink; 2 hours fits a + * 5 GiB repository ZIP at about 6 Mbit/s. headersTimeout keeps its default, and the per-upload + * stall timeout stops a slow-drip client from holding a connection that long. */ -export const API_SERVER_OPTIONS = { requestTimeout: 30 * 60 * 1000 }; +export const API_SERVER_OPTIONS = { requestTimeout: 2 * 60 * 60 * 1000 }; /** An upload that sends no bytes for this long is aborted and its partial file deleted. */ export const ARTIFACT_UPLOAD_STALL_TIMEOUT_MS = 60_000; diff --git a/apps/web/e2e/media-pipeline.spec.ts b/apps/web/e2e/media-pipeline.spec.ts index eb63fe8..3f6ff70 100644 --- a/apps/web/e2e/media-pipeline.spec.ts +++ b/apps/web/e2e/media-pipeline.spec.ts @@ -80,3 +80,32 @@ test("an image in the main conversation opens and downloads from the artifact ro await download.click(); expect((await saved).suggestedFilename()).toBe("pixel.png"); }); + +test("a repository ZIP above the inline cap streams up as a file", async ({ page }, testInfo) => { + const stamp = Date.now(); + await signup(page, `zip-upload-${stamp}@rakazo.test`, "password12", "Zip Upload"); + await completeOnboarding(page); + + const input = page.locator('input[type="file"]'); + await expect(input).toHaveAttribute("accept", /application\/zip/); + const upload = page.waitForResponse( + (response) => + response.request().method() === "POST" && /\/api\/artifacts\?/.test(response.url()), + ); + // Windows reports ZIPs under the legacy type; 12 MiB is above the 10 MiB inline cap. + await input.setInputFiles({ + name: "repo.zip", + mimeType: "application/x-zip-compressed", + buffer: Buffer.alloc(12 * 1024 * 1024, 1), + }); + await expect(page.getByRole("button", { name: "Remove repo.zip" })).toBeVisible(); + await page.getByRole("combobox", { name: /Message/ }).fill("here is the repo"); + await page.keyboard.press("Enter"); + const response = await upload; + expect(response.status()).toBe(201); + expect(response.request().headers()["content-type"]).toBe("application/zip"); + await expect(page.getByTestId("transcript").getByText("repo.zip")).toBeVisible({ + timeout: 30_000, + }); + await captureScreenshot(page, testInfo, "zip-upload"); +}); diff --git a/apps/web/src/lib/artifact-upload.ts b/apps/web/src/lib/artifact-upload.ts index 12d11e4..4c4d192 100644 --- a/apps/web/src/lib/artifact-upload.ts +++ b/apps/web/src/lib/artifact-upload.ts @@ -1,8 +1,8 @@ import { t } from "@lingui/core/macro"; import { - ARTIFACT_MAX_BYTES_BY_KIND, ArtifactSchema, ATTACHMENT_MAX_BYTES, + artifactMaxBytes, attachmentKindOf, } from "@rakazo/contracts"; import type { ArtifactTarget } from "./artifact-open"; @@ -12,7 +12,7 @@ type UploadedArtifact = ReturnType; /** Largest file of this type the composer accepts. */ export function attachmentMaxBytes(mimeType: string): number { - return ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)]; + return artifactMaxBytes(mimeType); } /** Media and anything above the inline RPC cap upload as a raw stream instead of base64. */ diff --git a/apps/web/src/pages/shell/composer.tsx b/apps/web/src/pages/shell/composer.tsx index 6f21915..9275100 100644 --- a/apps/web/src/pages/shell/composer.tsx +++ b/apps/web/src/pages/shell/composer.tsx @@ -41,7 +41,8 @@ import { previewMessageText } from "./message-preview"; import type { PendingAttachment } from "./shell-types"; import { FALLBACK_BOT_COLOR } from "./shell-types"; -const ATTACHMENT_ACCEPT = ATTACHMENT_ALLOWED_MIME_TYPES.join(","); +// ".zip" too: Windows reports ZIPs as application/x-zip-compressed. +const ATTACHMENT_ACCEPT = [...ATTACHMENT_ALLOWED_MIME_TYPES, ".zip"].join(","); const addButtonClass = "tactile-press size-10 shrink-0 rounded-full border border-border bg-foreground/5 text-foreground/80 transition-all hover:bg-foreground/10 hover:text-foreground flex items-center justify-center cursor-pointer"; diff --git a/infra/sandboxes/supervisor/src/file-stream.docker.test.ts b/infra/sandboxes/supervisor/src/file-stream.docker.test.ts new file mode 100644 index 0000000..e40204e --- /dev/null +++ b/infra/sandboxes/supervisor/src/file-stream.docker.test.ts @@ -0,0 +1,48 @@ +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { Readable } from "node:stream"; +import Docker from "dockerode"; +import { expect, it } from "vitest"; +import { writeContainerFile } from "./index.js"; + +const MiB = 1024 * 1024; + +function chunks(count: number, failAfter?: number) { + return Readable.from( + (async function* () { + for (let index = 0; index < count; index += 1) { + if (index === failAfter) throw new Error("upload aborted"); + yield Buffer.alloc(MiB, index % 256); + } + })(), + ); +} + +// Opt-in: needs Docker and a local Python image, but no network or credentials. +it.skipIf(process.env.VERIFY_DOCKER_FILE_STREAM !== "1")( + "streams large files into a container and leaves nothing behind when the stream fails", + async () => { + const image = process.env.VERIFY_DOCKER_PYTHON_IMAGE ?? "python:3.12-alpine"; + const name = `rakazo-file-stream-test-${randomUUID()}`; + const docker = (...args: string[]) => execFileSync("docker", args, { encoding: "utf8" }); + docker("run", "-d", "--name", name, image, "sh", "-c", "mkdir -p /home/rakazo && sleep 600"); + try { + const container = new Docker().getContainer(name); + await writeContainerFile(container, "/home/rakazo/attachments/repo.zip", chunks(64)); + expect(docker("exec", name, "stat", "-c", "%s %a", "/home/rakazo/attachments/repo.zip")).toBe( + `${64 * MiB} 600\n`, + ); + + await expect( + writeContainerFile(container, "/home/rakazo/attachments/broken.zip", chunks(64, 8)), + ).rejects.toThrow("upload aborted"); + expect(docker("exec", name, "ls", "/home/rakazo/attachments")).toBe("repo.zip\n"); + + await writeContainerFile(container, "/home/rakazo/run.sh", Buffer.from("echo hi\n"), true); + expect(docker("exec", name, "stat", "-c", "%s %a", "/home/rakazo/run.sh")).toBe("8 700\n"); + } finally { + docker("rm", "-f", name); + } + }, + 120_000, +); diff --git a/infra/sandboxes/supervisor/src/index.test.ts b/infra/sandboxes/supervisor/src/index.test.ts index 59f2ee7..3b1b0cd 100644 --- a/infra/sandboxes/supervisor/src/index.test.ts +++ b/infra/sandboxes/supervisor/src/index.test.ts @@ -201,6 +201,12 @@ describe("sandbox supervisor HTTP boundary", () => { expect(supervisorRequestBodyLimit("POST", "/computers/id/files/")).toBe( MAX_SUPERVISOR_FILE_REQUEST_BYTES, ); + expect(supervisorRequestBodyLimit("POST", "/computers/id/files/stream")).toBe( + Number.POSITIVE_INFINITY, + ); + expect(supervisorRequestBodyLimit("PUT", "/computers/id/files/stream")).toBe( + MAX_SUPERVISOR_REQUEST_BYTES, + ); expect(supervisorRequestBodyLimit("PUT", "/computers/id/files")).toBe( MAX_SUPERVISOR_REQUEST_BYTES, ); diff --git a/infra/sandboxes/supervisor/src/index.ts b/infra/sandboxes/supervisor/src/index.ts index 64e8b53..765645e 100644 --- a/infra/sandboxes/supervisor/src/index.ts +++ b/infra/sandboxes/supervisor/src/index.ts @@ -3,6 +3,9 @@ import { existsSync } from "node:fs"; import { mkdir } from "node:fs/promises"; import http from "node:http"; import path from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream as NodeReadableStream } from "node:stream/web"; import { fileURLToPath } from "node:url"; import { serve } from "@hono/node-server"; import { @@ -103,14 +106,22 @@ const computerScreens = new Map>(); export const MAX_SUPERVISOR_REQUEST_BYTES = 1024 * 1024; export const MAX_SUPERVISOR_FILE_REQUEST_BYTES = 16 * 1024 * 1024 + 64 * 1024; -/** Keep normal control requests small while allowing the existing 16 MiB file payload. */ +/** + * Keep normal control requests small while allowing the existing 16 MiB file payload. Streamed + * file writes go straight to disk, so their size is bounded by the artifact that is copied. + */ export function supervisorRequestBodyLimit(method: string, pathname: string): number { - return method === "POST" && /^\/computers\/[^/]+\/files\/?$/.test(pathname) + if (method !== "POST") return MAX_SUPERVISOR_REQUEST_BYTES; + if (/^\/computers\/[^/]+\/files\/stream\/?$/.test(pathname)) return Number.POSITIVE_INFINITY; + return /^\/computers\/[^/]+\/files\/?$/.test(pathname) ? MAX_SUPERVISOR_FILE_REQUEST_BYTES : MAX_SUPERVISOR_REQUEST_BYTES; } const limitSupervisorRequestBody: MiddlewareHandler = async (c, next) => { + if (supervisorRequestBodyLimit(c.req.method, c.req.path) === Number.POSITIVE_INFINITY) { + return next(); + } return bodyLimit({ maxSize: supervisorRequestBodyLimit(c.req.method, c.req.path), onError: (context) => context.json({ error: "Request body is too large." }, 413), @@ -594,6 +605,31 @@ app.post("/computers/:id/files", async (c) => { } }); +app.post("/computers/:id/files/stream", async (c) => { + const body = c.req.raw.body; + if (!body) return c.json({ error: "file is empty" }, 400); + try { + const { container } = await managedContainer( + c.req.param("id"), + c.req.header("x-rakazo-bot-id"), + c.req.header("x-rakazo-space-id"), + ); + const relative = normalizeWorkspaceRelative(c.req.query("path") ?? ""); + if (!relative) throw new Error("file path is required"); + await writeContainerFile( + container, + workspaceTarget(relative), + Readable.fromWeb(body as NodeReadableStream), + c.req.query("executable") === "true", + ); + return c.json({ ok: true }); + } catch (error) { + await body.cancel().catch(() => undefined); + const message = error instanceof Error ? error.message : String(error); + return c.json({ error: message }, 400); + } +}); + app.get("/computers/:id/screen", async (c) => { const id = c.req.param("id"); try { @@ -803,7 +839,9 @@ function startSupervisor() { computerResourceLimits(); const port = Number(process.env.SUPERVISOR_PORT ?? 7091); const hostname = process.env.SUPERVISOR_HOST ?? "127.0.0.1"; - const server = serve({ fetch: app.fetch, hostname, port }, () => { + // Streamed file writes copy whole repository archives; Node's 5-minute default would cut them. + const serverOptions = { requestTimeout: 2 * 60 * 60 * 1000 }; + const server = serve({ fetch: app.fetch, hostname, port, serverOptions }, () => { logger.info("supervisor listening", { "http.host": hostname, "http.port": port }); }); let stopping = false; @@ -1402,21 +1440,57 @@ async function observeContainer(container: Docker.Container, display = ":1") { return parseObservation(result.stdout); } -async function writeContainerFile( +/** + * Writes beside the target and renames it into place, so a failed write never leaves a truncated + * file. A buffer is complete before the write starts and commits in the same exec; a stream only + * commits once it ended cleanly. Python copies stdin in chunks, so large streams stay off-heap. + */ +export async function writeContainerFile( container: Docker.Container, target: string, - content: Buffer, + content: Buffer | Readable, executable = false, ) { - const script = [ - "import os, sys", - "target = sys.argv[1]", - "os.makedirs(os.path.dirname(target), exist_ok=True)", - "with open(target, 'wb') as f: f.write(sys.stdin.buffer.read())", - `os.chmod(target, ${executable ? "0o700" : "0o600"})`, - ].join("\n"); + const partial = `${target}.part-${randomUUID()}`; + const commit = [ + `os.chmod(sys.argv[1], ${executable ? "0o700" : "0o600"})`, + "os.replace(sys.argv[1], sys.argv[2])", + ]; + const write = [ + "import os, shutil, sys", + "os.makedirs(os.path.dirname(sys.argv[1]), exist_ok=True)", + "with open(sys.argv[1], 'wb') as f: shutil.copyfileobj(sys.stdin.buffer, f, 1024 * 1024)", + ]; + if (Buffer.isBuffer(content)) { + await runContainerPython( + container, + [...write, ...commit].join("\n"), + [partial, target], + content, + ); + return; + } + try { + await runContainerPython(container, write.join("\n"), [partial], content); + } catch (error) { + await runContainerPython( + container, + "import os, sys\ntry: os.remove(sys.argv[1])\nexcept FileNotFoundError: pass", + [partial], + ).catch(() => undefined); + throw error; + } + await runContainerPython(container, ["import os, sys", ...commit].join("\n"), [partial, target]); +} + +async function runContainerPython( + container: Docker.Container, + script: string, + args: string[], + stdin: Buffer | Readable = Buffer.alloc(0), +) { const exec = await container.exec({ - Cmd: ["python3", "-c", script, target], + Cmd: ["python3", "-c", script, ...args], AttachStdin: true, AttachStdout: true, AttachStderr: true, @@ -1426,14 +1500,37 @@ async function writeContainerFile( const stream = await exec.start({ hijack: true, stdin: true }); const chunks: Buffer[] = []; stream.on("data", (data: Buffer) => chunks.push(data)); - stream.end(content); - await new Promise((resolve, reject) => { + const ended = new Promise((resolve, reject) => { stream.on("end", resolve); + stream.on("close", resolve); stream.on("error", reject); }); - const inspect = await exec.inspect(); + if (Buffer.isBuffer(stdin)) { + stream.end(stdin); + } else { + try { + await pipeline(stdin, stream); + } catch (error) { + // pipeline already destroyed the exec stream with this error; `ended` must not reject unseen. + ended.catch(() => undefined); + stream.destroy(); + await waitForExecExit(exec); + throw error; + } + } + await ended; + const inspect = await waitForExecExit(exec); if ((inspect.ExitCode ?? 0) !== 0) { const output = demuxDockerStream(Buffer.concat(chunks)); throw new Error(output.stderr || output.stdout || "file write failed"); } } + +/** Docker reports the exit code a moment after the attached stream closes. */ +async function waitForExecExit(exec: Docker.Exec) { + for (let attempt = 0; ; attempt += 1) { + const inspect = await exec.inspect(); + if (!inspect.Running || attempt >= 50) return inspect; + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} diff --git a/packages/adapters/src/artifact-files.ts b/packages/adapters/src/artifact-files.ts index b228bf1..bcede22 100644 --- a/packages/adapters/src/artifact-files.ts +++ b/packages/adapters/src/artifact-files.ts @@ -2,7 +2,7 @@ import { constants } from "node:fs"; import { open } from "node:fs/promises"; import type { AdapterContext, ArtifactStore, ArtifactStreamPut } from "@rakazo/adapter-kit"; import { ArtifactTooLargeError } from "@rakazo/adapter-kit"; -import { ARTIFACT_MAX_BYTES_BY_KIND, attachmentKindOf } from "@rakazo/contracts"; +import { artifactMaxBytes } from "@rakazo/contracts"; import { validateAttachmentMimeType } from "@rakazo/core"; import type { PrismaClient } from "@rakazo/db"; @@ -30,11 +30,6 @@ type StoreArtifactDeps = { artifacts: ArtifactStore; }; -/** Largest artifact of this MIME type that may be streamed or generated. */ -export function artifactMaxBytes(mimeType: string): number { - return ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)]; -} - /** * Streams bytes into the artifact store and records the artifact row; the stored blob is removed * again when the row cannot be written. Rejects disallowed MIME types and anything above the diff --git a/packages/adapters/src/computer-support.ts b/packages/adapters/src/computer-support.ts index 00948e0..7ab7afd 100644 --- a/packages/adapters/src/computer-support.ts +++ b/packages/adapters/src/computer-support.ts @@ -37,6 +37,13 @@ export function computerFileTransferMaxBytes(computer: Pick : REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES; } +/** Docker computers also take streamed writes, so files copied onto them have no transfer cap. */ +export function computerFileWriteMaxBytes(computer: Pick): number { + return computer.kind === "docker" + ? Number.POSITIVE_INFINITY + : computerFileTransferMaxBytes(computer); +} + /** Streams a file into the workspace; providers without streaming writes receive its bytes. */ export async function writeComputerFileStream( sandbox: SandboxProvider, diff --git a/packages/adapters/src/docker-sandbox.test.ts b/packages/adapters/src/docker-sandbox.test.ts index e4500de..0aab3d4 100644 --- a/packages/adapters/src/docker-sandbox.test.ts +++ b/packages/adapters/src/docker-sandbox.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; import type { ProcessEvent } from "@rakazo/adapter-kit"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -348,4 +350,42 @@ describe("Docker page browser", () => { }), ); }); + + it("streams file writes to the supervisor without buffering them", async () => { + const received: { url?: string; bytes: number; type?: string } = { bytes: 0 }; + const server = createServer((request, response) => { + received.url = request.url; + received.type = request.headers["content-type"]; + request.on("data", (chunk: Buffer) => { + received.bytes += chunk.byteLength; + }); + request.on("end", () => { + response.setHeader("content-type", "application/json"); + response.end('{"ok":true}'); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const { port } = server.address() as AddressInfo; + const provider = new DockerSandboxProvider(`http://127.0.0.1:${port}`, "test-token"); + const chunk = new Uint8Array(1024 * 1024); + await provider.writeFileStream( + { id: "computer", botId: "bot", kind: "docker", providerRef: "computer" }, + { + path: "attachments/repo.zip", + stream: (async function* () { + for (let index = 0; index < 8; index += 1) yield chunk; + })(), + }, + context, + ); + expect(received).toEqual({ + url: "/computers/computer/files/stream?path=attachments%2Frepo.zip", + bytes: 8 * 1024 * 1024, + type: "application/octet-stream", + }); + } finally { + await new Promise((resolve) => server.close(resolve)); + } + }); }); diff --git a/packages/adapters/src/docker-sandbox.ts b/packages/adapters/src/docker-sandbox.ts index 5300bf6..93cf8a9 100644 --- a/packages/adapters/src/docker-sandbox.ts +++ b/packages/adapters/src/docker-sandbox.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { Readable } from "node:stream"; import type { AdapterContext, CommandRequest, @@ -11,6 +12,7 @@ import type { PageBrowserCommand, PageBrowserResult, PortableFile, + PortableFileStream, ProcessEvent, SandboxProvider, ScreenRequest, @@ -18,6 +20,7 @@ import type { } from "@rakazo/adapter-kit"; import { boundedSandboxCommandTimeoutMs, resolveSupervisorToken } from "@rakazo/core"; import { outgoingCorrelationHeaders } from "@rakazo/logging"; +import { byteChunks } from "./byte-streams.js"; import { boundedComputerActions, clampRounded, @@ -397,6 +400,26 @@ export class DockerSandboxProvider implements SandboxProvider { if (!res.ok) throw new Error(`sandbox file write failed: ${res.status}`); } + /** Streams the body to the supervisor, which writes it to disk without buffering it. */ + async writeFileStream(computer: ComputerRef, file: PortableFileStream, context: AdapterContext) { + const query = new URLSearchParams({ path: normalizeWorkspacePath(file.path) }); + if (file.executable) query.set("executable", "true"); + const res = await fetch(this.url(`/computers/${computer.id}/files/stream?${query}`), { + method: "POST", + headers: { + ...this.headers(context, computer.botId), + "content-type": "application/octet-stream", + }, + body: Readable.toWeb(Readable.from(byteChunks(file.stream))) as unknown as BodyInit, + duplex: "half", + signal: context.signal, + } as RequestInit & { duplex: "half" }); + if (!res.ok) { + const detail = await safeBody(res, context.signal); + throw new Error(`sandbox file write failed: ${res.status} ${detail}`.trim()); + } + } + async *exportWorkspace(computer: ComputerRef, context: AdapterContext) { yield* this.walkWorkspace(computer, "", context); } diff --git a/packages/adapters/src/thread-artifacts.test.ts b/packages/adapters/src/thread-artifacts.test.ts index 075d14b..bc17168 100644 --- a/packages/adapters/src/thread-artifacts.test.ts +++ b/packages/adapters/src/thread-artifacts.test.ts @@ -227,7 +227,7 @@ describe("current-turn thread files", () => { const remote = await materializeCurrentTurnFiles(deps, blocks, { context, - computer: { id: "computer-1", botId: "bot-1", kind: "docker", providerRef: "docker-1" }, + computer: { id: "computer-1", botId: "bot-1", kind: "e2b", providerRef: "e2b-1" }, computerMode: "dedicated", }); expect(remote.map((file) => [file.artifactId, file.path])).toEqual([ @@ -254,6 +254,24 @@ describe("current-turn thread files", () => { ]); expect(writeFile).toHaveBeenCalledTimes(3); expect(currentTurnFilesInstruction(local)).not.toContain("too large"); + + // Docker computers take streamed writes, so large files reach them too. + const writeFileStream = vi.fn().mockResolvedValue(undefined); + const docker = await materializeCurrentTurnFiles( + { ...deps, sandbox: { writeFile, writeFileStream } as unknown as SandboxProvider }, + blocks, + { + context, + computer: { id: "computer-3", botId: "bot-1", kind: "docker", providerRef: "docker-1" }, + computerMode: "dedicated", + }, + ); + expect(docker.map((file) => file.path)).toEqual([ + "attachments/img-1.png", + "attachments/aud-1.mp3", + "attachments/vid-1.mp4", + ]); + expect(writeFileStream).toHaveBeenCalledTimes(3); }); it("skips loading when a turn has no attachments", async () => { diff --git a/packages/adapters/src/thread-artifacts.ts b/packages/adapters/src/thread-artifacts.ts index 0903f7b..0d1d68a 100644 --- a/packages/adapters/src/thread-artifacts.ts +++ b/packages/adapters/src/thread-artifacts.ts @@ -7,18 +7,18 @@ import type { } from "@rakazo/adapter-kit"; import { ArtifactTooLargeError } from "@rakazo/adapter-kit"; import type { ComputerMode, MessageBlock } from "@rakazo/contracts"; -import { ATTACHMENT_MAX_BYTES } from "@rakazo/contracts"; +import { ATTACHMENT_MAX_BYTES, artifactMaxBytes } from "@rakazo/contracts"; import { attachmentExtensionForMimeType, inferAttachmentMimeType, messageBlockForArtifact, } from "@rakazo/core"; import type { PrismaClient } from "@rakazo/db"; -import { artifactMaxBytes, storeArtifactStream } from "./artifact-files.js"; +import { storeArtifactStream } from "./artifact-files.js"; import { discardByteStream } from "./byte-streams.js"; import { assertTeamWorkspacePathAllowed, - computerFileTransferMaxBytes, + computerFileWriteMaxBytes, resolveBotWorkspacePath, writeComputerFileStream, } from "./computer-support.js"; @@ -141,7 +141,7 @@ export async function materializeCurrentTurnFiles( }, }); const byId = new Map(rows.map((row) => [row.id, row])); - const transferMaxBytes = computerFileTransferMaxBytes(input.computer); + const transferMaxBytes = computerFileWriteMaxBytes(input.computer); const materialized: MaterializedThreadFile[] = []; for (const block of attachmentBlocks) { diff --git a/packages/contracts/src/attachments.test.ts b/packages/contracts/src/attachments.test.ts index fb593cc..3e1a58a 100644 --- a/packages/contracts/src/attachments.test.ts +++ b/packages/contracts/src/attachments.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest"; -import { attachmentKindOf, MessageBlock, validateThreadsSendInput } from "./index.js"; +import { + ARCHIVE_MAX_BYTES, + ARTIFACT_MAX_BYTES_BY_KIND, + artifactMaxBytes, + attachmentKindOf, + MessageBlock, + validateThreadsSendInput, +} from "./index.js"; describe("attachment contracts", () => { it("parses image and file message blocks", () => { @@ -22,6 +29,14 @@ describe("attachment contracts", () => { ).toMatchObject({ kind: "file", size: 1234 }); }); + it("gives ZIP archives a far larger cap than other files", () => { + expect(attachmentKindOf("application/zip")).toBe("file"); + expect(artifactMaxBytes("application/zip")).toBe(ARCHIVE_MAX_BYTES); + expect(ARCHIVE_MAX_BYTES).toBe(5 * 1024 ** 3); + expect(artifactMaxBytes("application/pdf")).toBe(ARTIFACT_MAX_BYTES_BY_KIND.file); + expect(artifactMaxBytes("video/mp4")).toBe(ARTIFACT_MAX_BYTES_BY_KIND.video); + }); + it("requires text or attachments for threads.send", () => { expect(validateThreadsSendInput({ text: "hello" })).toBe(true); expect(validateThreadsSendInput({ artifactIds: ["art_1"] })).toBe(true); diff --git a/packages/contracts/src/attachments.ts b/packages/contracts/src/attachments.ts index 3434e35..75a6b57 100644 --- a/packages/contracts/src/attachments.ts +++ b/packages/contracts/src/attachments.ts @@ -28,6 +28,7 @@ export const ATTACHMENT_FILE_MIME_TYPES = [ "application/json", "text/vtt", "application/x-subrip", + "application/zip", ] as const; export const ATTACHMENT_ALLOWED_MIME_TYPES = [ @@ -50,6 +51,16 @@ export const ARTIFACT_MAX_BYTES_BY_KIND: Record = { file: 25 * 1024 * 1024, }; +/** A ZIP carries a whole project or repository, so it gets a far larger cap than other files. */ +export const ARCHIVE_MAX_BYTES = 5 * 1024 * 1024 * 1024; + +/** Largest artifact of this MIME type that may be streamed or generated. */ +export function artifactMaxBytes(mimeType: string): number { + return mimeType === "application/zip" + ? ARCHIVE_MAX_BYTES + : ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)]; +} + export type AttachmentMimeType = (typeof ATTACHMENT_ALLOWED_MIME_TYPES)[number]; export function isAttachmentImageMimeType(mimeType: string): boolean { diff --git a/packages/core/src/attachments.test.ts b/packages/core/src/attachments.test.ts index 8d3baac..22ae382 100644 --- a/packages/core/src/attachments.test.ts +++ b/packages/core/src/attachments.test.ts @@ -13,7 +13,10 @@ import { describe("attachment helpers", () => { it("rejects unsupported mime types and empty payloads", () => { - expect(() => validateAttachmentMimeType("application/zip")).toThrow(AttachmentValidationError); + expect(() => validateAttachmentMimeType("application/x-msdownload")).toThrow( + AttachmentValidationError, + ); + expect(() => validateAttachmentMimeType("application/zip")).not.toThrow(); expect(() => decodeAttachmentBase64("")).toThrow(AttachmentValidationError); expect(() => decodeAttachmentBase64("aGVsbG8=trailing-junk")).toThrow( AttachmentValidationError, @@ -53,7 +56,12 @@ describe("attachment helpers", () => { expect(inferAttachmentMimeType("notes.md", "")).toBe("text/markdown"); expect(inferAttachmentMimeType("notes.markdown", "text/plain")).toBe("text/markdown"); expect(inferAttachmentMimeType("notes.md", "application/pdf")).toBe("application/pdf"); - expect(inferAttachmentMimeType("archive.zip", "")).toBeNull(); + expect(inferAttachmentMimeType("repo.zip", "")).toBe("application/zip"); + // Windows reports ZIPs under a legacy type. + expect(inferAttachmentMimeType("repo.ZIP", "application/x-zip-compressed")).toBe( + "application/zip", + ); + expect(inferAttachmentMimeType("archive.tar", "")).toBeNull(); }); it("scopes current-turn images to user-triggered runs", () => { diff --git a/packages/core/src/attachments.ts b/packages/core/src/attachments.ts index 4aea966..5905e3d 100644 --- a/packages/core/src/attachments.ts +++ b/packages/core/src/attachments.ts @@ -182,6 +182,7 @@ const EXTENSION_MIME_TYPES: Record = { ".oga": "audio/ogg", ".vtt": "text/vtt", ".srt": "application/x-subrip", + ".zip": "application/zip", }; const MIME_TYPE_EXTENSIONS: Record = { @@ -204,6 +205,7 @@ const MIME_TYPE_EXTENSIONS: Record = { "audio/ogg": ".ogg", "text/vtt": ".vtt", "application/x-subrip": ".srt", + "application/zip": ".zip", }; export function inferAttachmentMimeType( diff --git a/packages/testkit/src/attachments.test.ts b/packages/testkit/src/attachments.test.ts index e6a47cb..9ff4b41 100644 --- a/packages/testkit/src/attachments.test.ts +++ b/packages/testkit/src/attachments.test.ts @@ -94,8 +94,8 @@ describeAttachments("chat attachments", () => { const badMime = await raw(app, cookie, "artifacts/create", { botId: bot.id, - name: "evil.zip", - mimeType: "application/zip", + name: "evil.exe", + mimeType: "application/x-msdownload", contentBase64: Buffer.from("zip").toString("base64"), }); expect(badMime.status).toBeGreaterThanOrEqual(400);