// biome-ignore-all lint/suspicious/noTemplateCurlyInString: Shell and GitHub expressions are literal workflow fixtures. import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync, } from "node:fs"; import os from "node:os"; import path from "node:path"; import { PNG } from "pngjs"; import { afterEach, describe, expect, it } from "vitest"; const repoRoot = path.resolve(import.meta.dirname, "../../.."); const extractor = path.join(repoRoot, "scripts/extract-playwright-artifacts.py"); const workflow = readFileSync( path.join(repoRoot, ".github/workflows/publish-playwright-report.yml"), "utf8", ); const temporaryDirectories: string[] = []; const screenshot = PNG.sync.write({ width: 1, height: 1, data: Buffer.from([23, 45, 67, 255]), }); afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { rmSync(directory, { recursive: true, force: true }); } }); /** Allocate a fixture directory that afterEach removes, including failed test runs. */ function temporaryDirectory() { const directory = mkdtempSync(path.join(os.tmpdir(), "playwright-publication-")); temporaryDirectories.push(directory); return directory; } /** Write a fixture file, creating its parent directories as needed. */ function write(file: string, content: string | Buffer) { mkdirSync(path.dirname(file), { recursive: true }); writeFileSync(file, content); } /** Create an offline ZIP fixture with optional file types keyed by entry name. */ function makeArchive( root: string, entries: Record, modes: Record = {}, ) { const archive = path.join(root, "artifact.zip"); execFileSync( "python3", [ "-c", `import base64, json, sys, zipfile modes = json.loads(sys.argv[2]) with zipfile.ZipFile(sys.argv[1], "w") as archive: for name, data in json.load(sys.stdin).items(): entry = zipfile.ZipInfo(name) entry.create_system = 3 entry.external_attr = modes.get(name, 0o100644) << 16 archive.writestr(entry, base64.b64decode(data)) `, archive, JSON.stringify(modes), ], { input: JSON.stringify( Object.fromEntries( Object.entries(entries).map(([name, data]) => [ name, Buffer.from(data).toString("base64"), ]), ), ), }, ); return archive; } describe("Playwright artifact publication boundary", () => { it("keeps extraction and explicit report inputs outside the trusted checkout", () => { expect(workflow).not.toContain("actions/download-artifact@"); expect(workflow).toContain( 'gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}/zip" > "$RUNNER_TEMP/playwright-artifacts.zip"', ); expect(workflow).toContain( 'python3 scripts/extract-playwright-artifacts.py \\\n "$RUNNER_TEMP/playwright-artifacts.zip" "$RUNNER_TEMP/playwright-artifacts"', ); expect(workflow).toContain( "PLAYWRIGHT_REPORT_DIR: ${{ runner.temp }}/playwright-artifacts/playwright-report", ); expect(workflow).toContain( "PLAYWRIGHT_TEST_RESULTS_DIR: ${{ runner.temp }}/playwright-artifacts/apps/web/test-results", ); expect(workflow).toContain("ref: ${{ github.event.repository.default_branch }}"); expect(workflow).toContain("persist-credentials: false"); expect(workflow).toContain("run: bash scripts/publish-playwright-report.sh"); expect(workflow).not.toContain("working-directory:"); }); it.each([ ["../escape.txt", 0o100644], ["nested/../../escape.txt", 0o100644], ["/escape.txt", 0o100644], ["nested\\..\\escape.txt", 0o100644], ["apps/web/test-results/link", 0o120777], ])("rejects unsafe ZIP entry %s before extracting any data", (name, mode) => { const root = temporaryDirectory(); const archive = makeArchive( root, { "playwright-report/index.html": "valid", [name]: "unsafe" }, { [name]: mode }, ); const destination = path.join(root, "artifacts"); expect(() => execFileSync("python3", [extractor, archive, destination], { stdio: "pipe" }), ).toThrow(); expect(existsSync(destination)).toBe(false); expect(existsSync(path.join(root, "escape.txt"))).toBe(false); }); it("refuses an existing destination, including a filesystem link", () => { const root = temporaryDirectory(); const destination = path.join(root, "artifacts"); symlinkSync(root, destination, "dir"); const archive = makeArchive(root, { "overwrite.txt": "unsafe" }); expect(() => execFileSync("python3", [extractor, archive, destination], { stdio: "pipe" }), ).toThrow(); expect(existsSync(path.join(root, "overwrite.txt"))).toBe(false); }); it.each(["pull_request", "push", "timed_out"])( "publishes %s through trusted code with fake storage and credentials", (scenario) => { const root = temporaryDirectory(); const checkout = path.join(root, "checkout"); const runnerTemp = path.join(root, "runner-temp"); const artifacts = path.join(runnerTemp, "playwright-artifacts"); const hasArtifact = scenario !== "timed_out"; const isPr = scenario !== "push"; const trustedFiles = [ "package.json", "scripts/publish-playwright-report.sh", "packages/testkit/src/cli/generate-playwright-report-dashboard.ts", "packages/testkit/src/cli/build-playwright-pr-screenshot-comment.ts", "packages/testkit/src/playwright-report-dashboard.ts", "packages/testkit/src/playwright-pr-screenshot-comment.ts", "packages/testkit/src/png-validation.ts", ]; for (const file of trustedFiles) { write(path.join(checkout, file), readFileSync(path.join(repoRoot, file))); } if (hasArtifact) { const replacements = Object.fromEntries( [ ...trustedFiles, "tsconfig.json", ".npmrc", "node_modules/tsx/package.json", ".tmp/playwright-dashboard/screenshots/review.json", ].map((file) => [file, "untrusted replacement"]), ); replacements["scripts/publish-playwright-report.sh"] = 'printf "%s" "$AWS_SECRET_ACCESS_KEY" > "$ATTACK_MARKER"; exit 90'; replacements["packages/testkit/src/cli/build-playwright-pr-screenshot-comment.ts"] = 'import { writeFileSync } from "node:fs"; writeFileSync(process.env.ATTACK_MARKER, process.env.GH_TOKEN);'; const archive = makeArchive(root, { ...replacements, "playwright-report/index.html": "Example report", "apps/web/test-results/example-chromium/checkpoint.png": screenshot, }); execFileSync("python3", [extractor, archive, artifacts]); expect(readFileSync(path.join(artifacts, "package.json"), "utf8")).toBe( "untrusted replacement", ); } // Dependencies are trusted and remain outside the extraction tree. for (const directory of ["node_modules", "packages/testkit/node_modules"]) { symlinkSync(path.join(repoRoot, directory), path.join(checkout, directory), "dir"); } const bin = path.join(root, "bin"); write( path.join(bin, "aws"), `#!/usr/bin/env python3 import os, pathlib, shutil, sys operation, source, destination = sys.argv[2:5] if source.startswith("s3://"): print("NoSuchKey 404", file=sys.stderr) sys.exit(1) target = pathlib.Path(os.environ["FAKE_STORAGE"]) / destination.removeprefix("s3://example-bucket/") target.parent.mkdir(parents=True, exist_ok=True) if operation == "sync": shutil.copytree(source, target, dirs_exist_ok=True) else: shutil.copyfile(source, target) `, ); execFileSync("chmod", ["+x", path.join(bin, "aws")]); const storage = path.join(root, "published"); const env = { PATH: `${bin}${path.delimiter}${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH}`, GITHUB_WORKSPACE: checkout, GITHUB_OUTPUT: path.join(root, "outputs"), AWS_ACCESS_KEY_ID: "fake-access-key", AWS_SECRET_ACCESS_KEY: "fake-storage-secret", S3_BUCKET: "example-bucket", S3_ENDPOINT: "https://storage.example.invalid", PLAYWRIGHT_PUBLIC_BASE_URL: "https://reports.example.invalid/playwright", PLAYWRIGHT_RESULT: scenario === "timed_out" ? "failure" : "success", PLAYWRIGHT_RUN_ATTEMPT: "1", PLAYWRIGHT_RUN_ID: "200", PLAYWRIGHT_RUN_NUMBER: "10", PLAYWRIGHT_RUN_URL: "https://example.invalid/actions/runs/200", PLAYWRIGHT_SHA: "a".repeat(40), PLAYWRIGHT_EVENT: isPr ? "pull_request" : "push", PLAYWRIGHT_BRANCH: isPr ? "example-change" : "main", PLAYWRIGHT_PR_NUMBER: isPr ? "42" : "", PLAYWRIGHT_REPOSITORY_URL: "https://example.invalid/project", PLAYWRIGHT_PUBLISH_REPORT: String(!isPr), PLAYWRIGHT_REPORT_DIR: path.join(artifacts, "playwright-report"), PLAYWRIGHT_TEST_RESULTS_DIR: path.join(artifacts, "apps/web/test-results"), FAKE_STORAGE: storage, ATTACK_MARKER: path.join(root, "executed-untrusted-code"), }; execFileSync("bash", ["scripts/publish-playwright-report.sh"], { cwd: checkout, env, stdio: "pipe", }); const published = path.join(storage, "playwright"); const run = path.join(published, "runs/200-1"); expect(existsSync(path.join(run, "report/index.html"))).toBe(!isPr); const manifest = JSON.parse( readFileSync(path.join(run, "screenshots/manifest.json"), "utf8"), ); expect(manifest.screenshots).toHaveLength(hasArtifact ? 1 : 0); if (hasArtifact) { expect(readFileSync(path.join(run, "screenshots/images/001-checkpoint.png"))).toEqual( screenshot, ); } if (isPr) { expect(readFileSync(env.GITHUB_OUTPUT, "utf8")).toContain("latest_pr_run=true"); const changedPaths = path.join(root, "changed-paths.txt"); write(changedPaths, "apps/web/src/example.tsx\n"); const comment = execFileSync( "pnpm", [ "exec", "tsx", "packages/testkit/src/cli/build-playwright-pr-screenshot-comment.ts", path.join(checkout, ".tmp/playwright-dashboard/screenshots/review.json"), changedPaths, ], { cwd: checkout, encoding: "utf8", env: { ...env, GH_TOKEN: "fake-comment-token", PLAYWRIGHT_GALLERY_URL: "https://reports.example.invalid/playwright/prs/42/index.html", PLAYWRIGHT_DASHBOARD_URL: "https://reports.example.invalid/playwright/index.html", }, }, ); expect(comment).toContain(""); expect(existsSync(path.join(published, "prs/42/index.html"))).toBe(true); } expect(existsSync(env.ATTACK_MARKER)).toBe(false); for (const file of trustedFiles) { expect(readFileSync(path.join(checkout, file))).toEqual( readFileSync(path.join(repoRoot, file)), ); } }, ); });