scripts/ci-scope.test.ts
Rama main · réplica de lectura del repositorio original.
import { execFile, execFileSync, spawnSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import { tmpdir } from "node:os";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
import { parse } from "yaml";
import {
ALL_OS, AREA_CHECKS, ciOutputs, FULL_LANE_EVENTS, FULL_SCOPE, laneOf, LINUX_OS, MAIN_SCOPE, moveLinuxIntoStatic, QUEUED_SCOPE, SELECTABLE_JOBS,
selectCiScope,
selectPushScope, SRC_OUTSIDE_RENDERER, versionOnlyPackageJson, VITEST_SUBSETS, vitestMatrix, WINDOWS_LABEL, WINDOWS_OS, windowsReason,
} from "./ci-scope.mjs";
import { HOST_E2E } from "./testing/host-e2e.ts";
const ROOT = fileURLToPath(new URL("..", import.meta.url));
const script = fileURLToPath(new URL("./ci-scope.mjs", import.meta.url));
const ALL = ["macos-latest", "ubuntu-latest", "windows-latest"];
const PR = ["ubuntu-latest", "windows-latest"];
const LINUX = ["ubuntu-latest"];
const MAIN = ["ubuntu-latest", "windows-latest"];
const SERVER_JOBS = ["build", "vitest", "behavior-evals", "packaged-server", "electron-smokes", "foss", "ui-smoke", "browser-identity", "package-linux", "browser-driver"];
const temporaryDirectories: string[] = [];
afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) rmSync(directory, { recursive: true, force: true });
});
function fixture(source = "src/page.ts") {
const directory = mkdtempSync(join(tmpdir(), "omb-ci-scope-"));
temporaryDirectories.push(directory);
const git = (...args: string[]) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
git("init", "--quiet");
git("config", "user.name", "CI fixture");
git("config", "user.email", "ci@example.invalid");
git("config", "commit.gpgsign", "false");
mkdirSync(dirname(join(directory, source)), { recursive: true });
mkdirSync(join(directory, "docs"), { recursive: true });
writeFileSync(join(directory, source), "source\n");
git("add", "--", source);
git("commit", "--quiet", "-m", "base");
return { directory, git, base: git("rev-parse", "HEAD") };
}
function run(directory: string, event: string, payload: unknown) {
const eventPath = join(directory, "event.json");
const outputPath = join(directory, "output.txt");
writeFileSync(eventPath, JSON.stringify(payload));
const result = spawnSync(process.execPath, [script], {
cwd: directory,
env: { ...process.env, GITHUB_EVENT_NAME: event, GITHUB_EVENT_PATH: eventPath, GITHUB_OUTPUT: outputPath },
encoding: "utf8",
timeout: 5_000,
});
expect(result.error).toBeUndefined();
expect(result.status, result.stderr).toBe(0);
return { output: outputs(readFileSync(outputPath, "utf8")), stderr: result.stderr };
}
/** The GITHUB_OUTPUT lines ci-scope wrote, each value parsed. */
function outputs(text: string) {
return Object.fromEntries(text.trimEnd().split("\n").map((line) => {
const at = line.indexOf("=");
return [line.slice(0, at), JSON.parse(line.slice(at + 1))];
}));
}
const packaged = (oses: string[]) => oses.filter((os) => os !== "macos-latest");
const everything = (oses: string[]) => ({ jobs: SELECTABLE_JOBS, vitest_os: oses, packaged_server_os: packaged(oses), vitest_matrix: vitestMatrix(oses) });
const staticOnly = (oses: string[], build: boolean) => ({ jobs: build ? ["build"] : [], vitest_os: oses, packaged_server_os: packaged(oses), vitest_matrix: vitestMatrix(oses) });
/** What a push to main and a merge group run: Linux's and Windows' shards, the macOS smokes and the short Linux checks. */
const MAIN_JOBS = ["build", "vitest", "behavior-evals", "packaged-server", "electron-smokes", "foss", "control-plane", "ui-smoke", "browser-identity"];
const mainLane = () => ({ jobs: MAIN_JOBS, vitest_os: MAIN, packaged_server_os: MAIN, vitest_matrix: vitestMatrix(MAIN) });
describe("vitest matrix", () => {
it("gives Linux and Windows eight shards each and macOS four, and only Linux every e2e file", () => {
const { include } = vitestMatrix(["macos-latest", "ubuntu-latest", "windows-latest"]);
const count = (os: string) => include.filter((entry) => entry.os === os).length;
expect([count("ubuntu-latest"), count("windows-latest"), count("macos-latest")]).toEqual([8, 8, 4]);
expect(include.find((entry) => entry.os === "windows-latest" && entry.shard === 8))
.toEqual({ os: "windows-latest", part: "shard 8/8", shard: 8, shards: 8, files: "", e2e: "host", node_tests: false, workers: 2 });
expect(include.filter((entry) => entry.node_tests).map((entry) => [entry.os, entry.shard]))
.toEqual([["macos-latest", 1], ["ubuntu-latest", 1], ["windows-latest", 1]]);
expect(include.filter((entry) => entry.e2e === "all").every((entry) => entry.os === "ubuntu-latest")).toBe(true);
});
it("keeps a pull request's shards off macOS", () => {
expect(vitestMatrix(["ubuntu-latest", "windows-latest"]).include.some((entry) => entry.os === "macos-latest")).toBe(false);
});
});
const checks = (...areas: string[]) => [...new Set(areas.flatMap((area) => AREA_CHECKS[area as keyof typeof AREA_CHECKS]))].sort();
describe("CI path scope", () => {
it("runs only the docs tests, without the build, for documentation, its evidence and the docs site", () => {
expect(selectCiScope([
"README.md", "AGENTS.md", "docs/guide.md", "docs/nested/guide.md", ".github/FUNDING.yml", "apps/docs/content/index.mdx",
"docs/verification/evidence/mentions/rtl-dom.json", "docs/screenshots/home.png",
])).toEqual({ areas: ["docs"], checks: ["docs-tests"] });
});
it("runs a test under scripts/ wherever the suite runs, without the build", () => {
expect(selectCiScope(["scripts/generate-locale.test.mjs", "scripts/testing/control-omb-ui.e2e.test.ts"]))
.toEqual({ areas: ["script-tests"], checks: ["browser-identity", "host-tests", "linux-tests", "ui-smoke"] });
});
it("runs the build, the Linux shards and the renderer smoke for a renderer change", () => {
expect(selectCiScope(["src/App.tsx", "public/icon.svg", "cloud-buy-preview.html", "src/locales/en.json", "docs/guide.md"]))
.toEqual({ areas: ["docs", "web"], checks: ["build", "docs-tests", "linux-tests", "ui-smoke"] });
});
it("runs the locale tests alone for translations, their tests and their hashes", () => {
expect(selectCiScope(["src/locales/de.json", "src/locales/source-hashes.json", "src/locales/words.test.ts"]))
.toEqual({ areas: ["locales"], checks: ["build", "locale-tests"] });
expect(selectCiScope(["src/locales/index.ts"]).areas).toEqual(["web"]);
});
it.each([
["server/index.ts"], ["shared/types.ts"], ["companion/src/index.ts"], ["enterprise/license.ts"], ["evals/runners/run-evals.ts"],
["skills/pdf/SKILL.md"], ["cloudflare/composio-broker/src/index.ts"], ["Dockerfile"], ["deploy/podman/run.sh"],
...SRC_OUTSIDE_RENDERER.map((file) => [file]),
])("runs every check that runs the server for %j", (file) => {
expect(selectCiScope([file])).toEqual({ areas: ["server"], checks: checks("server") });
});
it.each([["electron/cua.mjs"], ["electron/shared-computer-access.mjs"], ["electron/capabilities.cjs"], ["electron/screen-status.mjs"]])(
"runs the Windows computer-control smokes too for %j, which they start as the installed app does", (file) => {
expect(selectCiScope([file])).toEqual({ areas: ["windows-computer-control"], checks: checks("server", "windows-computer-control") });
expect(selectCiScope([file]).checks).toContain("windows-cua");
});
it.each([["electron/main.mjs"], ["build/icon.icns"], ["electron-builder.yml"], ["index.html"], ["rail.html"]])("runs the server's checks for the desktop's %j", (file) => {
expect(selectCiScope([file])).toEqual({ areas: ["desktop"], checks: checks("server") });
});
it("runs nothing beyond static, which validates them, for the announcements file, its schema and pictures", () => {
expect(selectCiScope(["announcements/announcements.json", "announcements/announcements.schema.json", "announcements/images/phone.png"]))
.toEqual({ areas: ["announcements"], checks: [] });
expect(selectCiScope(["announcements/announcements.json", "docs/announcements.md"])).toEqual({ areas: ["announcements", "docs"], checks: ["docs-tests"] });
});
it("runs only the control-plane job for the control plane", () => {
expect(selectCiScope(["cloudflare/control-plane/src/index.ts"])).toEqual({ areas: ["control-plane"], checks: ["control-plane"] });
});
it.each([["relay/src/server.ts"], ["relay/pnpm-lock.yaml"], ["relay/Dockerfile"], ["relay/deploy/Caddyfile"], ["relay/README.md"]])(
"runs only the relay job for the relay's %j", (file) => {
expect(selectCiScope([file])).toEqual({ areas: ["relay"], checks: ["relay"] });
});
it("runs the server's checks and the relay's for the shared E2E vectors, which the relay keeps a pinned copy of", () => {
expect(selectCiScope(["companion/src/e2e/vectors/v1.json"])).toEqual({ areas: ["e2e-vectors"], checks: [...checks("server"), "relay"].sort() });
expect(selectCiScope(["companion/src/e2e/noise.ts"])).toEqual({ areas: ["server"], checks: checks("server") });
});
it("selects every area's checks for a change across areas", () => {
expect(selectCiScope(["src/App.tsx", "src/locales/de.json", "cloudflare/control-plane/src/index.ts"]))
.toEqual({ areas: ["control-plane", "locales", "web"], checks: ["build", "control-plane", "linux-tests", "locale-tests", "ui-smoke"] });
});
it("keeps filenames containing newlines as one path that can affect anything", () => {
expect(selectCiScope(["src/first.md\nsecond.md"])).toEqual(FULL_SCOPE);
expect(selectCiScope(["first\nsecond.md"])).toEqual(FULL_SCOPE);
});
it.each([
[], [""], ["docs/../server/file.md"], ["./src/App.tsx"],
[".github/workflows/ci.yml"], ["scripts/ci-scope.mjs"], ["scripts/testing/host-e2e.ts"], [".gitattributes"],
["package.json"], ["pnpm-lock.yaml"], ["pnpm-workspace.yaml"], ["tsconfig.json"], ["vite.config.ts"], [".oxlintrc.json"],
["third_party/browser/agent-browser-windows-stdio.patch"], ["scripts/testing/ci-retry-runner.ts"], ["unknown/file"], ["src/App.tsx", "package.json"],
])("runs everything for %j", (...files) => {
expect(selectCiScope(files)).toEqual(FULL_SCOPE);
});
it("lists every check in some area and every check of an area in the full scope", () => {
const named = new Set(Object.values(AREA_CHECKS).flat());
for (const job of SELECTABLE_JOBS.filter((job) => job !== "vitest")) expect(named.has(job), job).toBe(true);
for (const check of named) expect(check in VITEST_SUBSETS || FULL_SCOPE.checks.includes(check), check).toBe(true);
});
});
describe("CI outputs", () => {
it("runs a renderer change's tests on Linux only", () => {
const out = ciOutputs(selectCiScope(["src/App.tsx"]), PR);
expect(out).toEqual({ jobs: ["build", "vitest", "ui-smoke"], vitest_os: PR, packaged_server_os: PR, vitest_matrix: vitestMatrix(["ubuntu-latest"]) });
});
it("runs a server change's tests on Linux and Windows, and its other checks", () => {
expect(ciOutputs(selectCiScope(["server/index.ts"]), PR)).toEqual({ jobs: SERVER_JOBS, vitest_os: PR, packaged_server_os: PR, vitest_matrix: vitestMatrix(PR) });
});
it("runs the locale tests in their own shards, with the Electron node tests, when no full Linux run covers them", () => {
const out = ciOutputs(selectCiScope(["src/locales/de.json"]), PR);
expect(out.jobs).toEqual(["build", "vitest"]);
expect(VITEST_SUBSETS["locale-tests"].files).toEqual([
"src/",
"electron/cua-launch.test.mjs",
"electron/cua-watch-restart.test.mjs",
"electron/cua-windows-isolation.test.mjs",
"electron/menu.test.mjs",
"electron/updater.test.mjs",
"scripts/generate-locale.test.mjs",
"scripts/testing/bot-tools-ui.e2e.test.ts",
"scripts/testing/key-readback.e2e.test.ts",
"scripts/testing/one-workspace-ui.e2e.test.ts",
"scripts/testing/rail-profile-ui.e2e.test.ts",
"scripts/testing/secret-card-descriptions.test.ts",
"scripts/testing/team-own-chat.e2e.test.ts",
"scripts/testing/team-side-switch-ui.e2e.test.ts",
"server/body-thread-guards.e2e.test.ts",
"server/client-notifications.e2e.test.ts",
"server/cloud-connections.e2e.test.ts",
"server/cloud-lending-browser.e2e.test.ts",
"server/cloud-lending-chrome.e2e.test.ts",
"server/cloud-lending-memory.e2e.test.ts",
"server/cloud-lending.e2e.test.ts",
"server/cloud-lent-mcp.e2e.test.ts",
"server/cloud-send-kinds.e2e.test.ts",
"server/cloud-send-move.e2e.test.ts",
"server/cloud-send-rooms.e2e.test.ts",
"server/cloud-send-routine-merge.e2e.test.ts",
"server/cloud-send-store.e2e.test.ts",
"server/cloud-send.e2e.test.ts",
"server/cloud-sync.e2e.test.ts",
"server/cloud-team-asker-owns.e2e.test.ts",
"server/cloud-team-away.e2e.test.ts",
"server/cloud-team-browser.e2e.test.ts",
"server/cloud-team-connections.e2e.test.ts",
"server/cloud-team-engine-homes.e2e.test.ts",
"server/cloud-team-gateway-mcp.e2e.test.ts",
"server/cloud-team-inbox.e2e.test.ts",
"server/cloud-team-mausbot-reach.e2e.test.ts",
"server/cloud-team-new-chat-private.e2e.test.ts",
"server/cloud-team-open-reach.e2e.test.ts",
"server/cloud-team-own-key-apps.e2e.test.ts",
"server/cloud-team-owners.e2e.test.ts",
"server/cloud-team-people.e2e.test.ts",
"server/cloud-team-per-person.e2e.test.ts",
"server/cloud-team-phones.e2e.test.ts",
"server/cloud-team-pictures.e2e.test.ts",
"server/cloud-team-presence.e2e.test.ts",
"server/cloud-team-privacy-holes.e2e.test.ts",
"server/cloud-team-private.e2e.test.ts",
"server/cloud-team-profile.e2e.test.ts",
"server/cloud-team-requests.e2e.test.ts",
"server/cloud-team-routines.e2e.test.ts",
"server/cloud-team-setup-request.e2e.test.ts",
"server/cloud-team.e2e.test.ts",
"server/host/brief-words.test.ts",
"server/host/lending-lifts.test.ts",
"server/lent-folders.e2e.test.ts",
"server/move-chosen-bots.e2e.test.ts",
"server/one-home-keep.e2e.test.ts",
"server/one-home-outdated.e2e.test.ts",
"server/one-home-retry.e2e.test.ts",
"server/one-home-session.e2e.test.ts",
"server/one-home-trial-key.e2e.test.ts",
"server/one-home-two-desks.e2e.test.ts",
"server/one-home.e2e.test.ts",
"server/parked-cards.e2e.test.ts",
"server/question-cards.e2e.test.ts",
"server/shared-computers.e2e.test.ts",
"server/shared-computers.gate.test.ts",
"server/team-move.e2e.test.ts",
"server/team-welcome.e2e.test.ts",
"server/use-my-mac.e2e.test.ts",
"server/workspace-banner.e2e.test.ts",
"scripts/server-locale-plugin.test.ts",
"server/server-lines.test.ts",
"server/host/devs-reports.test.ts",
"server/automation-reports.test.ts",
"server/incidents.test.ts",
"server/routine-run-card.test.ts",
"server/host/chief-notices-retry.test.ts",
"server/connector-card-text.test.ts",
"server/host/secret-cards.test.ts",
"server/notify.test.ts",
"server/room-lines.test.ts",
"server/post-to-room.test.ts",
"server/teammate-lines.test.ts",
"server/chat-notices.test.ts",
"server/recall-disclosure.test.ts",
"server/routine-cron-text.test.ts",
"server/routine-requests.test.ts",
"server/cloud-send-arrived.test.ts",
"server/cloud-team-bot-access.e2e.test.ts",
"server/cloud-team-one-answer.e2e.test.ts",
"server/cloud-team-preference.e2e.test.ts",
"server/cloud-team-rules.e2e.test.ts",
"server/cloud-team-rules-running-turn.e2e.test.ts",
"server/outside-work-threads.e2e.test.ts",
"server/own-home.e2e.test.ts",
"server/places-elsewhere.e2e.test.ts",
"server/question-routing.e2e.test.ts",
"server/team-protocol.e2e.test.ts",
"server/team-small-fixes.e2e.test.ts",
"server/team-mausbot-followup.e2e.test.ts",
"server/team-no-ai.e2e.test.ts",
"server/team-refused-handoff.e2e.test.ts",
"server/two-places.e2e.test.ts",
"scripts/ci-workflow.test.ts",
"scripts/pre-push-check.test.ts",
"server/host/wait-words.test.ts",
"server/place-inbox.e2e.test.ts",
"server/bot-api-team.e2e.test.ts",
"server/cloud-team-first-look.e2e.test.ts",
"server/cloud-team-mausbot-own-chat.e2e.test.ts",
"server/connector-stayed.e2e.test.ts",
"server/email/host.test.ts",
"server/host/outside-lineage.test.ts",
"server/memory-move.test.ts",
"server/outside-lineage.e2e.test.ts",
"server/same-name-mentions.e2e.test.ts",
"server/skill-switch.test.ts",
]);
expect(out.vitest_matrix.include).toEqual([1, 2].map((shard) => ({
os: "ubuntu-latest", part: `locale tests ${shard}/2`, shard, shards: 2,
files: VITEST_SUBSETS["locale-tests"].files.join(" "), e2e: "all", node_tests: shard === 1, workers: 4,
})));
expect(ciOutputs(selectCiScope(["src/locales/de.json", "src/App.tsx"]), PR).vitest_matrix).toEqual(vitestMatrix(["ubuntu-latest"]));
});
it("runs the docs tests in one job for docs, and skips the build", () => {
const out = ciOutputs(selectCiScope(["docs/analytics.md"]), PR);
expect(out).toMatchObject({ jobs: ["vitest"] });
expect(out.vitest_matrix.include).toEqual([{
os: "ubuntu-latest", part: "docs tests", shard: 1, shards: 1, files: VITEST_SUBSETS["docs-tests"].files.join(" "), e2e: "all", node_tests: false, workers: 4,
}]);
expect(ciOutputs(selectCiScope(["docs/analytics.md", "src/locales/de.json"]), PR).vitest_matrix.include.map((entry: { part: string }) => entry.part))
.toEqual(["locale tests 1/2", "locale tests 2/2", "docs tests"]);
});
it("skips the build and every job when nothing is selected, keeping a full matrix for the skipped vitest job", () => {
expect(ciOutputs({ checks: [] }, PR)).toEqual(staticOnly(PR, false));
});
it("runs every job, on every platform the event covers, for the full scope", () => {
expect(ciOutputs(FULL_SCOPE, ALL)).toEqual(everything(ALL));
expect(ciOutputs(FULL_SCOPE, PR)).toEqual(everything(PR));
});
it("puts every e2e-capable platform's shards in the vitest job only when its tests are selected", () => {
const windows = ciOutputs({ checks: ["host-tests"] }, ALL).vitest_matrix.include;
expect(new Set(windows.map((entry: { os: string }) => entry.os))).toEqual(new Set(["macos-latest", "windows-latest"]));
});
it("runs Linux's and Windows' every shard, the macOS smokes and the short Linux checks for main", () => {
const out = ciOutputs(MAIN_SCOPE, MAIN);
expect(out).toEqual(mainLane());
expect(out.vitest_matrix.include.map((entry: { os: string }) => entry.os)).toEqual([...Array(8).fill("ubuntu-latest"), ...Array(8).fill("windows-latest")]);
expect(out.packaged_server_os).toEqual(MAIN);
for (const job of ["windows-cua", "package-linux", "browser-driver", "cloudflared-linux-arm64"]) expect(out.jobs, job).not.toContain(job);
expect(MAIN_SCOPE.checks.every((check) => FULL_SCOPE.checks.includes(check))).toBe(true);
});
it("leaves every Windows job out where the run's platforms leave Windows out", () => {
const out = ciOutputs(FULL_SCOPE, LINUX);
expect(out.jobs).toEqual(SELECTABLE_JOBS.filter((job) => job !== "windows-cua"));
expect(out.vitest_matrix).toEqual(vitestMatrix(LINUX));
expect(out.packaged_server_os).toEqual(LINUX);
expect(out.vitest_os).toEqual(LINUX);
expect(ciOutputs(FULL_SCOPE, WINDOWS_OS).jobs).toContain("windows-cua");
expect(LINUX_OS).toEqual(LINUX);
expect(WINDOWS_OS).toEqual(PR);
});
it("runs only the broker's control-plane checks for a commit the merge queue already tested", () => {
expect(ciOutputs(QUEUED_SCOPE, MAIN)).toEqual({ ...staticOnly(MAIN, false), jobs: ["control-plane"] });
});
it("names each event's lane: a pull request its areas, main and the merge queue Linux, the rest every platform", () => {
expect(laneOf("pull_request")).toBe("pr");
expect(laneOf("push")).toBe("main");
expect(laneOf("merge_group")).toBe("main");
for (const event of FULL_LANE_EVENTS) expect(laneOf(event), event).toBe("full");
expect(FULL_LANE_EVENTS).toEqual(["schedule", "workflow_dispatch"]);
expect(ALL_OS).toEqual(ALL);
});
});
describe("the Linux work in the static job, on a Boat VM", () => {
const linuxShards = vitestMatrix(LINUX).include;
it("runs a server change's Linux shards, behavior evals and packaged server there, and no Linux job", () => {
const out = moveLinuxIntoStatic(ciOutputs(selectCiScope(["server/index.ts"]), LINUX));
expect(out.jobs).toEqual(SERVER_JOBS.filter((job) => !["vitest", "behavior-evals", "packaged-server"].includes(job)));
expect(out.linux_lanes).toEqual({ vitest: linuxShards, evals: true, packaged_server: true });
expect(out.vitest_matrix).toEqual(vitestMatrix(LINUX));
expect(out.vitest_os).toEqual(LINUX);
});
it("keeps the Windows shards and packaged server as jobs", () => {
const out = moveLinuxIntoStatic(ciOutputs(FULL_SCOPE, PR));
expect(out.jobs).toEqual(SELECTABLE_JOBS.filter((job) => job !== "behavior-evals"));
expect(out.vitest_matrix).toEqual(vitestMatrix(["windows-latest"]));
expect(out.packaged_server_os).toEqual(["windows-latest"]);
expect(out.linux_lanes).toEqual({ vitest: linuxShards, evals: true, packaged_server: true });
expect(moveLinuxIntoStatic(ciOutputs(MAIN_SCOPE, MAIN)).jobs).toEqual(MAIN_JOBS.filter((job) => job !== "behavior-evals"));
});
it("runs a docs or translation change's subset there, and nothing when nothing is selected", () => {
const docs = moveLinuxIntoStatic(ciOutputs({ checks: ["docs-tests"] }, LINUX));
expect(docs.jobs).toEqual([]);
expect(docs.linux_lanes).toEqual({ vitest: ciOutputs({ checks: ["docs-tests"] }, LINUX).vitest_matrix.include, evals: false, packaged_server: false });
expect(moveLinuxIntoStatic(ciOutputs({ checks: [] }, LINUX))).toEqual({
...staticOnly(LINUX, false), linux_lanes: { vitest: [], evals: false, packaged_server: false },
});
});
it("does it only when the static job is told it runs on Boat", () => {
const { directory, git, base } = fixture("src/App.tsx");
writeFileSync(join(directory, "src/App.tsx"), "changed\n");
git("commit", "--quiet", "-am", "renderer");
const eventPath = join(directory, "event.json");
const outputPath = join(directory, "output.txt");
writeFileSync(eventPath, JSON.stringify({ pull_request: { base: { sha: base }, head: { sha: git("rev-parse", "HEAD") } } }));
const result = spawnSync(process.execPath, [script], {
cwd: directory, encoding: "utf8", timeout: 5_000,
env: { ...process.env, GITHUB_EVENT_NAME: "pull_request", GITHUB_EVENT_PATH: eventPath, GITHUB_OUTPUT: outputPath, OMB_CI_LINUX_IN_STATIC: "true" },
});
expect(result.status, result.stderr).toBe(0);
expect(outputs(readFileSync(outputPath, "utf8"))).toEqual(moveLinuxIntoStatic(ciOutputs(selectCiScope(["src/App.tsx"]), LINUX)));
expect(result.stdout).toContain("this job runs 8 Linux vitest lane(s)");
});
});
describe("Windows on a pull request", () => {
const reason = (files: string[], lines: string[] = [], labels: string[] = []) =>
windowsReason({ files, labels, hostE2e: Object.keys(HOST_E2E), lines: () => lines });
it("runs for the windows label whatever changed", () => {
expect(WINDOWS_LABEL).toBe("windows");
expect(reason(["docs/guide.md"], [], ["bug", "windows"])).toBe('the "windows" label');
expect(reason(["server/index.ts"], [], ["Windows-ish"])).toBeNull();
});
it.each([
"electron/main.mjs", "build/installer.nsh", "electron-builder.yml", "maus.ps1", "scripts/build-cua-win.mjs",
"scripts/smoke-cua-win-embedded.mjs", "server/win32-paths.ts", "server/shell/windows.ts", "server/foo.win32.test.ts",
".github/workflows/package-win.yml", "package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", ".gitattributes", "vite.config.ts",
"third_party/browser/agent-browser-windows-stdio.patch", ".github/workflows/ci.yml", "scripts/ci-scope.mjs",
"scripts/testing/host-e2e.ts", "scripts/testing/vitest-shard-weights.json", "server/testing/host-timeout.ts",
"scripts/prepare-cloudflared.mjs", "scripts/after-pack.mjs", ...Object.keys(HOST_E2E),
])("runs for %s", (file) => {
expect(reason(["src/App.tsx", file])).toBe(file);
});
it.each([
"server/index.ts", "src/components/WindowFrame.tsx", "server/darwin.ts", "server/twin.ts", "scripts/verify-sidebar.ts",
"scripts/testing/rail-preview.tsx", "scripts/control-omb.ts", ".github/workflows/boat.yml", "tsconfig.json", "server/delta-context.e2e.test.ts",
])("waits for main for %s alone", (file) => {
expect(reason([file])).toBeNull();
});
it.each([
['+ if (process.platform === "win32") return;'], ["- // Windows keeps a running process's folder"], ["+ const shell = \"powershell.exe\";"],
["+ join(env.USERPROFILE, x)"], ["+ spawn(cmd, args, { windowsHide: true })"], ["+ path.win32.join(a, b)"], ["+ runs-on: windows-latest"],
])("runs for a changed line %j", (line) => {
expect(reason(["server/index.ts"], ["diff --git a/server/index.ts b/server/index.ts", line])).toBe(`the changed line ${JSON.stringify(line)}`);
});
it("reads only added and removed lines, not file headers or a window", () => {
expect(reason(["server/index.ts"], [
"--- a/server/Windows.ts", "+++ b/server/Windows.ts", "@@ -1 +1 @@ Windows", " Windows context", "+const frame = window.open(url);",
"- new BrowserWindow()", "+// two windows side by side",
])).toBeNull();
});
it("reads the lines only when no label or path decided, and runs Windows when they cannot be read", () => {
const unread = () => { throw new Error("diff too large"); };
expect(windowsReason({ files: ["electron/main.mjs"], lines: unread })).toBe("electron/main.mjs");
expect(windowsReason({ files: ["server/index.ts"], lines: unread })).toBe("a change whose lines could not be read (diff too large)");
});
});
/** Relative imports, `@/` included, that load code when run; `import type` loads nothing. */
const IMPORT = /(?:^|[^\w$.])(?:import|export)\s+(?!type\b)(?:[^"'`;]*?\sfrom\s*)?["']([^"']+)["']|(?:^|[^\w$.])(?:import|require)\s*\(\s*["']([^"']+)["']\s*\)/gm;
const CODE = /\.[cm]?[jt]sx?$/;
const TEST_FILE = /\.(?:test|node-test)\.[cm]?[jt]sx?$|(?:^|\/)(?:test|testing|fixtures)\//;
function codeFiles(directory: string): string[] {
if (!existsSync(directory)) return [];
return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
if (entry.name === "node_modules" || entry.name.startsWith(".")) return [];
const path = join(directory, entry.name);
return entry.isDirectory() ? codeFiles(path) : CODE.test(entry.name) ? [path] : [];
});
}
function resolveImport(from: string, specifier: string) {
const base = (specifier.startsWith("@/") ? join(ROOT, "src", specifier.slice(2)) : specifier.startsWith(".") ? resolve(dirname(from), specifier) : null)
?.replace(/\?.*$/, "");
if (!base) return null;
const candidates = [base, base.replace(/\.js$/, ".ts"), ...["ts", "tsx", "mjs", "cjs", "js"].map((extension) => `${base}.${extension}`),
...["ts", "tsx", "js"].map((extension) => join(base, `index.${extension}`))];
return candidates.find((candidate) => existsSync(candidate) && statSync(candidate).isFile()) ?? null;
}
const importsOf = new Map<string, string[]>();
function imports(file: string) {
if (!importsOf.has(file)) {
importsOf.set(file, CODE.test(file)
? [...readFileSync(file, "utf8").matchAll(IMPORT)].flatMap((match) => resolveImport(file, match[1] ?? match[2]) ?? [])
: []);
}
return importsOf.get(file)!;
}
/** Every repository file the given files load, transitively, themselves included. */
function importClosure(entries: string[]) {
const seen = new Set<string>();
const pending = [...entries];
while (pending.length > 0) {
const file = pending.pop()!;
if (seen.has(file)) continue;
seen.add(file);
pending.push(...imports(file).filter((target) => !seen.has(target)));
}
return [...seen].map((file) => relative(ROOT, file).split("\\").join("/"));
}
describe("the repository's import graph", () => {
it("lists every renderer file the server, Electron, the companion, evals, the workers or the macOS frame smoke load", () => {
const frameSmoke = readFileSync(join(ROOT, "scripts/smoke-app-frames.mjs"), "utf8");
expect(frameSmoke).toContain('resolveDir: join(ROOT, "src/components/app")');
expect(frameSmoke).toContain("from \\\"./app-frame-host.ts\\\"");
const runtime = ["server", "shared", "electron", "companion", "enterprise", "evals", "cloudflare"]
.flatMap((directory) => codeFiles(join(ROOT, directory)))
.filter((file) => !TEST_FILE.test(relative(ROOT, file).split("\\").join("/")));
expect(runtime.length).toBeGreaterThan(500);
// The language packs the server reads its lines from stay locale changes (SRC_OUTSIDE_RENDERER says why).
const loaded = importClosure([...runtime, join(ROOT, "src/components/app/app-frame-host.ts")])
.filter((file) => file.startsWith("src/") && !/^src\/locales\/[^/]+\.json$/.test(file));
expect(loaded.sort()).toEqual([...SRC_OUTSIDE_RENDERER].sort());
});
it("runs every test outside the renderer that reads a translation other than English with the locale tests", () => {
const files = ["server", "shared", "electron", "companion", "enterprise", "evals", "scripts", "cloudflare"]
.flatMap((directory) => codeFiles(join(ROOT, directory)))
// The selection names the catalogs without reading them.
.filter((file) => !/[/\\]ci-scope\.(?:mjs|test\.ts)$/.test(file));
const mentionsTranslations = (file: string) => {
const text = readFileSync(file, "utf8");
// A browser extension's own _locales folder is not one of ours.
return [...text.matchAll(/(?<!_)locales/g)].some((match) => !/^locales\W{1,6}en\.json/.test(text.slice(match.index, match.index + 24)));
};
const readers = new Set(files.filter(mentionsTranslations));
const affected = files
.filter((file) => /\.test\.[cm]?[jt]s$/.test(file))
.filter((file) => importClosure([file]).some((loaded) => /^src\/locales\/(?!en\.json$)/.test(loaded) || readers.has(join(ROOT, loaded))))
.map((file) => relative(ROOT, file).split("\\").join("/"));
expect(affected).toContain("scripts/generate-locale.test.mjs");
expect(affected).toContain("server/host/brief-words.test.ts");
// The Electron node tests, which compare their own copy of some strings with every catalog, run on the locale tests' first shard.
expect(affected.filter((file) => !VITEST_SUBSETS["locale-tests"].files.some((filter) => file.includes(filter)))).toEqual([]);
});
it("runs every test that reads the docs with the docs tests or in static", () => {
const ci = parse(readFileSync(join(ROOT, ".github/workflows/ci.yml"), "utf8"));
const inStatic = (ci.jobs.static.steps as { run?: string }[]).flatMap((step) => step.run?.startsWith("pnpm exec vitest run ") ? step.run.split(" ").slice(4) : []);
expect(inStatic).toContain("scripts/testing/verification-docs.test.ts");
// A docs path a test reads from the repository root: join(ROOT, "docs", ...), new URL("docs/...", ROOT) or read("docs/...").
const DOCS_READ = /(?:(?:join|resolve)\(\s*ROOT\s*,\s*|new URL\(\s*|\bread\w*\(\s*)["'`]((?:apps\/docs|docs)\b[^"'`]*|[A-Z][A-Z_]+\.md)["'`]/g;
const readers = ["server", "shared", "electron", "companion", "enterprise", "evals", "scripts", "src", "cloudflare"]
.flatMap((directory) => codeFiles(join(ROOT, directory)))
.filter((file) => /\.(?:test|node-test)\.[cm]?[jt]sx?$/.test(file) && !file.endsWith("ci-scope.test.ts"))
.filter((file) => [...readFileSync(file, "utf8").replace(/(["'`])\s*,\s*(["'`])/g, "/").matchAll(DOCS_READ)]
.some((match) => existsSync(join(ROOT, match[1]))))
.map((file) => relative(ROOT, file).split("\\").join("/"))
.sort();
expect(readers).toEqual([...VITEST_SUBSETS["docs-tests"].files, ...inStatic.filter((file) => file === "scripts/testing/verification-docs.test.ts")].sort());
});
it("never loads a test file from other code, so a test under scripts/ affects only the jobs that run it", () => {
const code = ["server", "shared", "electron", "companion", "enterprise", "evals", "scripts", "src", "cloudflare"]
.flatMap((directory) => codeFiles(join(ROOT, directory)));
const loaders = code.filter((file) => imports(file).some((target) => /\.test\.[cm]?[jt]sx?$/.test(target)));
expect(loaders.map((file) => relative(ROOT, file))).toEqual([]);
});
});
describe("CI scope CLI", () => {
it("runs only the docs tests, without the build, for a docs-only pull request", () => {
const { directory, git, base } = fixture("docs/guide.md");
writeFileSync(join(directory, "docs/guide.md"), "updated guide\n");
git("commit", "--quiet", "-am", "docs");
const { output, stderr } = run(directory, "pull_request", {
pull_request: { base: { sha: base }, head: { sha: git("rev-parse", "HEAD") } },
});
expect(output).toEqual(ciOutputs({ checks: ["docs-tests"] }, LINUX));
expect(output).toMatchObject({ jobs: ["vitest"] });
expect(stderr).toBe("");
});
it("ignores base-only server changes after the pull request diverges", () => {
const { directory, git, base: common } = fixture("server/original.ts");
writeFileSync(join(directory, "server/original.ts"), "base-only change\n");
git("commit", "--quiet", "-am", "base change");
const base = git("rev-parse", "HEAD");
git("checkout", "--quiet", "--detach", common);
writeFileSync(join(directory, "docs/guide.md"), "PR-only guide\n");
git("add", "--", "docs/guide.md");
git("commit", "--quiet", "-m", "PR docs");
const { output, stderr } = run(directory, "pull_request", {
pull_request: { base: { sha: base }, head: { sha: git("rev-parse", "HEAD") } },
});
expect(output).toEqual(ciOutputs({ checks: ["docs-tests"] }, LINUX));
expect(output).toMatchObject({ jobs: ["vitest"] });
expect(stderr).toBe("");
});
it.each(["delete", "rename"])("retains the original source path on %s", (change) => {
const source = "server/original.ts";
const { directory, git, base } = fixture(source);
if (change === "delete") git("rm", "--", source);
else git("mv", "--", source, "docs/renamed.md");
git("commit", "--quiet", "-m", change);
const { output, stderr } = run(directory, "pull_request", {
pull_request: { base: { sha: base }, head: { sha: git("rev-parse", "HEAD") } },
});
// A server change that cannot break Windows first: its Windows jobs wait for main.
expect(output).toEqual({ jobs: SERVER_JOBS, vitest_os: LINUX, packaged_server_os: LINUX, vitest_matrix: vitestMatrix(LINUX) });
expect(stderr).toBe("");
});
it("runs a renderer pull request's tests on Linux, and a translation pull request's locale tests", () => {
for (const [source, jobs, oses] of [["src/App.tsx", ["build", "vitest", "ui-smoke"], ["ubuntu-latest"]], ["src/locales/de.json", ["build", "vitest"], []]] as const) {
const { directory, git, base } = fixture(source);
writeFileSync(join(directory, source), "changed\n");
git("commit", "--quiet", "-am", "change");
const { output, stderr } = run(directory, "pull_request", {
pull_request: { base: { sha: base }, head: { sha: git("rev-parse", "HEAD") } },
});
expect(output).toEqual(ciOutputs(selectCiScope([source]), LINUX));
expect(output.jobs).toEqual(jobs);
expect(output.vitest_matrix.include.filter((entry: { files: string }) => !entry.files).map((entry: { os: string }) => entry.os))
.toEqual(oses.flatMap((os) => Array(8).fill(os)));
expect(stderr).toBe("");
}
});
it("reads filenames with spaces without turning source into docs", () => {
const source = "src/first second.md";
const { directory, git, base } = fixture(source);
writeFileSync(join(directory, source), "changed\n");
git("commit", "--quiet", "-am", "change");
const { output } = run(directory, "pull_request", {
pull_request: { base: { sha: base }, head: { sha: git("rev-parse", "HEAD") } },
});
expect(output).toEqual(ciOutputs({ checks: AREA_CHECKS.web }, LINUX));
});
it.each(["missing", "invalid", "unavailable", "empty"])("falls back loudly for a %s diff", (kind) => {
const { directory, base } = fixture();
const payload = kind === "missing" ? {} : {
pull_request: { base: { sha: base }, head: { sha: kind === "invalid" ? "--help" : kind === "unavailable" ? "f".repeat(40) : base } },
};
const { output, stderr } = run(directory, "pull_request", payload);
expect(output).toEqual(everything(ALL));
expect(stderr).toContain("using all checks");
});
it.each(["schedule", "workflow_dispatch"])("runs everything on every platform for %s", (event) => {
const { directory } = fixture("docs/guide.md");
expect(run(directory, event, {}).output).toEqual(everything(ALL));
});
it("runs main's checks for a merge group, whatever it changes", () => {
const { directory } = fixture("docs/guide.md");
expect(run(directory, "merge_group", { merge_group: { head_sha: "f".repeat(40) } }).output).toEqual(mainLane());
});
});
const pkg = (version: string, extra: Record<string, unknown> = {}) =>
`${JSON.stringify({ name: "openmausbot", private: true, version, scripts: { test: "vitest run" }, ...extra }, null, 2)}\n`;
describe("version-only package.json", () => {
it("is only a changed top-level version", () => {
expect(versionOnlyPackageJson(pkg("0.1.100"), pkg("0.1.101"))).toBe(true);
// the bump writes package.json with npm pkg set; formatting is not a change
expect(versionOnlyPackageJson(pkg("0.1.100"), JSON.stringify(JSON.parse(pkg("0.1.101"))))).toBe(true);
});
it.each([
["the same version", pkg("0.1.100"), pkg("0.1.100")],
["a dependency added", pkg("0.1.100"), pkg("0.1.101", { dependencies: { zod: "4" } })],
["a script changed", pkg("0.1.100"), pkg("0.1.101").replace("vitest run", "vitest run --bail")],
["a nested version changed", pkg("0.1.100", { engines: { version: "1" } }), pkg("0.1.101", { engines: { version: "2" } })],
["other keys reordered", pkg("0.1.100"), JSON.stringify({ private: true, name: "openmausbot", version: "0.1.101", scripts: { test: "vitest run" } })],
["no version before", JSON.stringify({ name: "x" }), JSON.stringify({ name: "x", version: "1.0.0" })],
["not JSON", "{", pkg("0.1.101")],
["not an object", "[]", "[]"],
])("is false for %s", (_, before, after) => {
expect(versionOnlyPackageJson(before, after)).toBe(false);
});
it("scopes a push by its own files", () => {
const show = (rev: string) => (rev === "before" ? pkg("0.1.100") : pkg("0.1.101"));
expect(selectPushScope(["package.json"], show)).toEqual({ runtime: false });
expect(selectPushScope(["docs/ci.md", "README.md"], show)).toEqual({ runtime: false });
expect(selectPushScope(["package.json", "pnpm-lock.yaml"], show)).toEqual({ runtime: true });
expect(selectPushScope(["package.json", "docs/ci.md"], show)).toEqual({ runtime: true });
expect(selectPushScope(["server/index.ts"], show)).toEqual({ runtime: true });
expect(selectPushScope(["package.json"], () => pkg("0.1.100"))).toEqual({ runtime: true });
expect(selectPushScope(["package.json"], () => { throw new Error("missing"); })).toEqual({ runtime: true });
});
it("never narrows a push the way it narrows a pull request", () => {
const show = () => pkg("0.1.100");
for (const file of ["apps/docs/content/index.mdx", "src/locales/de.json", "src/App.tsx", "cloudflare/control-plane/src/index.ts", "server/index.ts"]) {
expect(selectPushScope([file], show), file).toEqual({ runtime: true });
}
expect(selectPushScope([], show)).toEqual({ runtime: true });
});
});
type FakeRun = { id?: number; event: string; status: string; conclusion: string | null };
/** A GitHub that answers the CI runs of one commit and the jobs of one run; the
* script under test runs asynchronously so this server can answer it. */
async function fakeRunsApi(runs: Record<string, FakeRun[]>, jobs: Record<number, Array<{ name: string; conclusion: string | null }>> = {}) {
const requests: string[] = [];
const server = createServer((request, response) => {
requests.push(`${request.method} ${request.url} ${request.headers.authorization}`);
const url = new URL(request.url!, "http://x");
const sha = url.searchParams.get("head_sha") ?? "";
const runId = /^\/repos\/o\/r\/actions\/runs\/(\d+)\/jobs$/.exec(url.pathname)?.[1];
response.setHeader("content-type", "application/json");
response.end(JSON.stringify(runId ? { jobs: jobs[Number(runId)] ?? [] } : { workflow_runs: runs[sha] ?? [] }));
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
return { url, requests, close: () => new Promise((resolve) => server.close(resolve)) };
}
async function runPush(directory: string, payload: unknown, env: Record<string, string | undefined>) {
const eventPath = join(directory, "event.json");
const outputPath = join(directory, "output.txt");
writeFileSync(eventPath, JSON.stringify(payload));
const childEnv: Record<string, string | undefined> = { ...process.env, GITHUB_EVENT_NAME: "push", GITHUB_EVENT_PATH: eventPath, GITHUB_OUTPUT: outputPath, ...env };
for (const [key, value] of Object.entries(childEnv)) if (value === undefined) delete childEnv[key];
const stderr = await new Promise<string>((resolve, reject) => {
execFile(process.execPath, [script], { cwd: directory, env: childEnv as NodeJS.ProcessEnv, encoding: "utf8", timeout: 10_000 },
(error, _stdout, stderr) => (error ? reject(new Error(`${error.message}\n${stderr}`)) : resolve(stderr)));
});
return { output: outputs(readFileSync(outputPath, "utf8")), stderr };
}
describe("CI scope CLI on a push to main", () => {
function pushFixture(change: (directory: string) => void) {
const { directory, git } = fixture("server/index.ts");
writeFileSync(join(directory, "package.json"), pkg("0.1.100"));
git("add", "--", "package.json");
git("commit", "--quiet", "-m", "package");
const before = git("rev-parse", "HEAD");
change(directory);
git("add", "--all");
git("commit", "--quiet", "-m", "change");
return { directory, before, after: git("rev-parse", "HEAD") };
}
const bump = (directory: string) => writeFileSync(join(directory, "package.json"), pkg("0.1.101"));
const docs = (directory: string) => writeFileSync(join(directory, "docs/guide.md"), "guide\n");
const code = (directory: string) => writeFileSync(join(directory, "server/index.ts"), "changed\n");
const green = [{ event: "push", status: "completed", conclusion: "success" }];
const runsOf = (sha: string) => `GET /repos/o/r/actions/workflows/ci.yml/runs?head_sha=${sha}&per_page=100 Bearer t`;
const github = (api: { url: string }) => ({ GITHUB_API_URL: api.url, GH_TOKEN: "t", GITHUB_REPOSITORY: "o/r" });
it.each([["the version bump", bump], ["a docs-only merge", docs]])("skips the runtime jobs for %s after a green commit", async (_, change) => {
const { directory, before, after } = pushFixture(change);
const api = await fakeRunsApi({ [before]: [{ event: "pull_request", status: "completed", conclusion: "failure" }, ...green] });
try {
const { output, stderr } = await runPush(directory, { before, after }, github(api));
expect(output).toEqual(staticOnly(MAIN, true));
expect(stderr).toBe("");
expect(api.requests).toEqual([runsOf(after), runsOf(before)]);
} finally {
await api.close();
}
});
it.each([["a code change", code], ["the version bump", bump]])("runs only the control-plane checks for %s the merge queue already passed", async (_, change) => {
const { directory, before, after } = pushFixture(change);
// The queue lands its group's commit once the gate passes, before the run's last job has finished.
const api = await fakeRunsApi({ [after]: [{ id: 77, event: "merge_group", status: "in_progress", conclusion: null }] }, {
77: [{ name: "vitest (ubuntu-latest, shard 1/8)", conclusion: "success" }, { name: "CI", conclusion: "success" }],
});
try {
const { output, stderr } = await runPush(directory, { before, after }, github(api));
expect(output).toEqual(ciOutputs(QUEUED_SCOPE, MAIN));
expect(stderr).toBe("");
} finally {
await api.close();
}
});
it.each([
["failed its gate", [{ name: "CI", conclusion: "failure" }]],
["has not passed its gate yet", [{ name: "CI", conclusion: null }]],
])("runs main's checks for a commit whose merge-group run %s", async (_, gateJobs) => {
const { directory, before, after } = pushFixture(code);
const api = await fakeRunsApi({ [after]: [{ id: 78, event: "merge_group", status: "completed", conclusion: "failure" }] }, { 78: gateJobs });
try {
const { output, stderr } = await runPush(directory, { before, after }, github(api));
expect(output).toEqual(mainLane());
expect(stderr).toBe("");
expect(api.requests).toEqual([runsOf(after), "GET /repos/o/r/actions/runs/78/jobs?per_page=100 Bearer t"]);
} finally {
await api.close();
}
});
it.each([
["was replaced by this push", [{ event: "push", status: "completed", conclusion: "cancelled" }]],
["failed", [{ event: "push", status: "completed", conclusion: "failure" }]],
["never ran", []],
["only passed as a pull request", [{ event: "pull_request", status: "completed", conclusion: "success" }]],
])("runs main's checks for a version bump when the commit before it %s", async (_, runs) => {
const { directory, before, after } = pushFixture(bump);
const api = await fakeRunsApi({ [before]: runs });
try {
const { output, stderr } = await runPush(directory, { before, after }, github(api));
expect(output).toEqual(mainLane());
expect(stderr).toContain("has no green CI run of its own");
} finally {
await api.close();
}
});
it("runs main's checks when the pushed commit's CI cannot be read", async () => {
const { directory, before, after } = pushFixture(bump);
const { output, stderr } = await runPush(directory, { before, after }, { GH_TOKEN: undefined, GITHUB_REPOSITORY: undefined });
expect(output).toEqual(mainLane());
expect(stderr).toContain("using main's checks");
});
it.each([
["a code change", code],
["a version bump with a dependency", (directory: string) => writeFileSync(join(directory, "package.json"), pkg("0.1.101", { dependencies: { zod: "4" } }))],
["a version bump with code", (directory: string) => { bump(directory); code(directory); }],
])("runs main's checks for %s, asking GitHub only about the merge queue", async (_, change) => {
const { directory, before, after } = pushFixture(change);
const api = await fakeRunsApi({ [before]: green });
try {
const { output, stderr } = await runPush(directory, { before, after }, github(api));
expect(output).toEqual(mainLane());
expect(stderr).toBe("");
expect(api.requests).toEqual([runsOf(after)]);
} finally {
await api.close();
}
});
it.each([
["a missing payload", {}],
["a new branch", { before: "0".repeat(40), after: "f".repeat(40) }],
["an unknown commit", { before: "f".repeat(40), after: "e".repeat(40) }],
])("falls back loudly to main's checks for %s", async (_, payload) => {
const { directory } = fixture();
const { output, stderr } = await runPush(directory, payload, {});
expect(output).toEqual(mainLane());
expect(stderr).toContain("using main's checks");
});
});
describe("CI scope CLI on a pull request's Windows jobs", () => {
function serverChange(content: string, file = "server/paths.ts") {
const { directory, git, base } = fixture(file);
writeFileSync(join(directory, file), content);
git("commit", "--quiet", "-am", "change");
return { directory, payload: { pull_request: { number: 7, base: { sha: base }, head: { sha: git("rev-parse", "HEAD") } } } };
}
/** A GitHub that answers one pull request's labels, or fails with `status`. */
async function fakeLabelsApi(labels: string[], status = 200) {
const requests: string[] = [];
const server = createServer((request, response) => {
requests.push(`${request.method} ${request.url} ${request.headers.authorization}`);
response.statusCode = status;
response.setHeader("content-type", "application/json");
response.end(JSON.stringify(status === 200 ? labels.map((name) => ({ name })) : { message: "nope" }));
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
return { env: { GITHUB_API_URL: url, GH_TOKEN: "t", GITHUB_REPOSITORY: "o/r" }, requests, close: () => new Promise((resolve) => server.close(resolve)) };
}
async function runPullRequest(directory: string, payload: unknown, env: Record<string, string> = {}) {
const eventPath = join(directory, "event.json");
const outputPath = join(directory, "output.txt");
writeFileSync(eventPath, JSON.stringify(payload));
const childEnv: NodeJS.ProcessEnv = { ...process.env, GITHUB_EVENT_NAME: "pull_request", GITHUB_EVENT_PATH: eventPath, GITHUB_OUTPUT: outputPath, ...env };
if (!env.GH_TOKEN) delete childEnv.GH_TOKEN;
const { stdout, stderr } = await new Promise<{ stdout: string; stderr: string }>((resolve, reject) => {
execFile(process.execPath, [script], { cwd: directory, env: childEnv, encoding: "utf8", timeout: 10_000 },
(error, out, err) => (error ? reject(new Error(`${error.message}\n${err}`)) : resolve({ stdout: out, stderr: err })));
});
return { output: outputs(readFileSync(outputPath, "utf8")), stdout, stderr };
}
const serverJobs = (oses: string[]) => ({ jobs: SERVER_JOBS, vitest_os: oses, packaged_server_os: oses, vitest_matrix: vitestMatrix(oses) });
it("runs the Windows jobs for a changed line that names Windows, and says why", async () => {
const { directory, payload } = serverChange('if (process.platform === "win32") quote();\n');
const { output, stdout, stderr } = await runPullRequest(directory, payload);
expect(output).toEqual(serverJobs(PR));
expect(stdout).toContain('Windows runs for the changed line "+if (process.platform === \\"win32\\") quote();"');
expect(stderr).toBe("");
});
it("leaves the Windows jobs to main for a change that cannot break Windows first, and says how to run them", async () => {
const { directory, payload } = serverChange("export const greeting = \"hello\";\n");
const { output, stdout, stderr } = await runPullRequest(directory, payload);
expect(output).toEqual(serverJobs(LINUX));
expect(stdout).toContain('label the PR "windows" and re-run');
expect(stderr).toBe("");
});
it("runs the Windows jobs for the desktop app", async () => {
const { directory, payload } = serverChange("app\n", "electron/main.mjs");
expect((await runPullRequest(directory, payload)).output).toEqual(serverJobs(PR));
});
it("runs the Windows jobs for the label the pull request has now, so a re-run after labelling runs them", async () => {
const { directory, payload } = serverChange("export const greeting = \"hello\";\n");
const api = await fakeLabelsApi(["bug", "windows"]);
try {
const { output, stderr } = await runPullRequest(directory, payload, api.env);
expect(output).toEqual(serverJobs(PR));
expect(stderr).toBe("");
expect(api.requests).toEqual(["GET /repos/o/r/issues/7/labels?per_page=100 Bearer t"]);
} finally {
await api.close();
}
});
it("falls back to the labels in the event when GitHub cannot answer", async () => {
const { directory, payload } = serverChange("export const greeting = \"hello\";\n");
const api = await fakeLabelsApi([], 502);
try {
const labelled = { pull_request: { ...payload.pull_request, labels: [{ name: "windows" }] } };
const { output, stderr } = await runPullRequest(directory, labelled, api.env);
expect(output).toEqual(serverJobs(PR));
expect(stderr).toContain("reading the pull request's labels from its event (HTTP 502");
} finally {
await api.close();
}
});
it("asks GitHub for no labels when the change selects no Windows job", async () => {
const { directory, git, base } = fixture("src/App.tsx");
writeFileSync(join(directory, "src/App.tsx"), "if (navigator.platform === \"win32\") {}\n");
git("commit", "--quiet", "-am", "renderer");
const api = await fakeLabelsApi(["windows"]);
try {
const payload = { pull_request: { number: 7, base: { sha: base }, head: { sha: git("rev-parse", "HEAD") } } };
const { output } = await runPullRequest(directory, payload, api.env);
expect(output).toEqual(ciOutputs(selectCiScope(["src/App.tsx"]), LINUX));
expect(api.requests).toEqual([]);
} finally {
await api.close();
}
});
});