scripts/nightly-report.mjs
Rama main · réplica de lectura del repositorio original.
import {
existsSync,
mkdirSync,
readFileSync,
readdirSync,
statSync,
writeFileSync,
} from "node:fs";
import { join, resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { summarize } from "../e2e/scripts/flaky.mjs";
const INFRASTRUCTURE = new Set([
"select",
"suite-contract",
"e2e-coverage",
"guard",
"published",
"report",
]);
export function minutesBetween(started, completed) {
if (started === null || completed === null) {
return 0;
}
const span = Date.parse(completed) - Date.parse(started);
return Math.round((span / 60000) * 10) / 10;
}
export function readJobs(jobs) {
const rows = [];
for (const job of jobs) {
if (INFRASTRUCTURE.has(job.name)) {
continue;
}
rows.push({
name: job.name,
conclusion: job.conclusion ?? "cancelled",
minutes: minutesBetween(job.started_at ?? null, job.completed_at ?? null),
});
}
rows.sort((left, right) => left.name.localeCompare(right.name));
return rows;
}
function filesUnder(dir, match) {
const found = [];
if (!existsSync(dir)) {
return found;
}
for (const entry of readdirSync(dir)) {
const path = join(dir, entry);
if (statSync(path).isDirectory()) {
found.push(...filesUnder(path, match));
continue;
}
if (match(entry)) {
found.push(path);
}
}
return found;
}
function jsonUnder(dir) {
return filesUnder(dir, (entry) => entry.endsWith(".json"));
}
export function collectFlaky(dir) {
const flaky = [];
for (const path of filesUnder(dir, (entry) => entry === "report.json")) {
const summary = summarize(JSON.parse(readFileSync(path, "utf8")));
flaky.push(...summary.flaky);
}
flaky.sort((left, right) =>
`${left.file}${left.title}`.localeCompare(`${right.file}${right.title}`),
);
return flaky;
}
export function collectMutation(dir) {
if (!existsSync(dir)) {
return null;
}
let killed = 0;
let lived = 0;
let uncovered = 0;
let uncoveredKnown = true;
let found = 0;
for (const path of jsonUnder(dir)) {
const report = JSON.parse(readFileSync(path, "utf8"));
if (
typeof report.mutants_killed !== "number" ||
typeof report.mutants_lived !== "number"
) {
continue;
}
killed += report.mutants_killed;
lived += report.mutants_lived;
if (
Number.isInteger(report.mutants_not_covered) &&
report.mutants_not_covered >= 0
) {
uncovered += report.mutants_not_covered;
} else {
uncoveredKnown = false;
}
found += 1;
}
if (found === 0) {
return null;
}
const total = killed + lived;
let score = 0;
if (total > 0) {
score = Math.round((killed / total) * 1000) / 10;
}
if (!uncoveredKnown) {
uncovered = null;
}
return { shards: found, killed, lived, uncovered, score };
}
export function collectSystemCoverage(dir) {
const reports = [];
const suites = new Set();
for (const path of filesUnder(dir, (entry) => entry === "summary.json")) {
const report = JSON.parse(readFileSync(path, "utf8"));
if (
typeof report.suite !== "string" ||
!/^[a-z][a-z0-9-]+$/.test(report.suite) ||
!Number.isSafeInteger(report.covered) ||
!Number.isSafeInteger(report.total) ||
report.covered < 0 ||
report.total <= 0 ||
report.covered > report.total
) {
throw new Error(`invalid system coverage report: ${path}`);
}
if (suites.has(report.suite)) {
throw new Error(`duplicate system coverage suite: ${report.suite}`);
}
suites.add(report.suite);
reports.push({
...report,
percent: Math.round((1000 * report.covered) / report.total) / 10,
});
}
return reports.sort((left, right) => left.suite.localeCompare(right.suite));
}
export function buildSummary(input) {
const rows = readJobs(input.jobs);
const failures = rows
.filter((row) => row.conclusion !== "success")
.map((row) => row.name);
const minutes =
Math.round(rows.reduce((total, row) => total + row.minutes, 0) * 10) / 10;
let systemCoverage = input.systemCoverage;
if (systemCoverage === undefined || systemCoverage === null) {
systemCoverage = [];
}
return {
sha: input.sha,
runUrl: input.runUrl,
trigger: input.trigger,
finishedAt: input.finishedAt,
jobs: { total: rows.length, failed: failures.length, minutes },
groups: rows,
failures,
flaky: input.flaky,
e2eCoverage: input.e2eCoverage,
mutation: input.mutation,
systemCoverage,
};
}
function delta(current, previous, unit = "") {
if (current === null || current === undefined) {
return "n/a";
}
const shown = `${String(current)}${unit}`;
if (previous === null || previous === undefined) {
return shown;
}
const change = Math.round((current - previous) * 10) / 10;
if (change === 0) {
return `${shown} (no change)`;
}
if (change > 0) {
return `${shown} (+${String(change)})`;
}
return `${shown} (${String(change)})`;
}
export function render(summary, previous) {
const older = previous ?? {};
const verdict = summary.failures.length === 0 ? "green" : "regression";
const newly = summary.failures.filter(
(name) => !(older.failures ?? []).includes(name),
);
const fixed = (older.failures ?? []).filter(
(name) => !summary.failures.includes(name),
);
const lines = [
`# Nightly ${summary.finishedAt.slice(0, 10)} — ${verdict}`,
"",
`Commit \`${summary.sha.slice(0, 12)}\`, triggered by ${summary.trigger}. [Run](${summary.runUrl})`,
"",
"| | this run | |",
"|---|---|---|",
`| jobs | ${String(summary.jobs.total)} | ${String(summary.jobs.failed)} failed |`,
`| job-minutes | ${delta(summary.jobs.minutes, older.jobs?.minutes)} | |`,
`| e2e coverage | ${delta(summary.e2eCoverage, older.e2eCoverage, "%")} | |`,
`| mutation score | ${delta(summary.mutation?.score ?? null, older.mutation?.score ?? null, "%")} | ${
summary.mutation === null || summary.mutation === undefined
? "not run"
: `${String(summary.mutation.killed)} killed, ${String(summary.mutation.lived)} lived`
} |`,
`| flaky | ${delta(summary.flaky.length, older.flaky?.length)} | passed only on retry |`,
];
const uncovered = summary.mutation?.uncovered;
if (uncovered === null || uncovered === undefined) {
lines.push(
"| uncovered mutants | n/a | absent from one or more package reports |",
);
} else {
lines.push(
`| uncovered mutants | ${delta(uncovered, older.mutation?.uncovered)} | records across reported build variants |`,
);
}
lines.push(
"",
"Mutation score is killed / (killed + lived). Uncovered mutants are excluded from that score; some are uninstrumented constant expressions.",
);
const systemCoverage = summary.systemCoverage;
if (
systemCoverage !== undefined &&
systemCoverage !== null &&
systemCoverage.length > 0
) {
lines.push(
"",
"## Separate system coverage",
"",
"These suites retain their own Go statement denominators. The browser E2E coverage above is unchanged by these profiles.",
"",
"| suite | covered / total statements | coverage |",
"|---|---|---|",
);
for (const report of systemCoverage) {
lines.push(
`| ${report.suite} | ${report.covered} / ${report.total} | ${report.percent}% |`,
);
}
}
if (newly.length > 0) {
lines.push("", "## New failures", "");
for (const name of newly) {
lines.push(`- ${name}`);
}
}
if (fixed.length > 0) {
lines.push("", "## Fixed since the last run", "");
for (const name of fixed) {
lines.push(`- ${name}`);
}
}
if (summary.failures.length > 0) {
lines.push("", "## Failing", "");
for (const name of summary.failures) {
lines.push(`- ${name}`);
}
}
if (summary.flaky.length > 0) {
lines.push(
"",
"## Flaky",
"",
"| test | file | project | attempts |",
"|---|---|---|---|",
);
for (const test of summary.flaky) {
lines.push(
`| ${test.title} | ${test.file}:${String(test.line)} | ${test.project} | ${String(test.attempts)} |`,
);
}
}
lines.push("", "## Jobs", "", "| job | result | minutes |", "|---|---|---|");
for (const row of summary.groups) {
lines.push(`| ${row.name} | ${row.conclusion} | ${String(row.minutes)} |`);
}
return `${lines.join("\n")}\n`;
}
export function verdictOf(summary) {
if (summary.failures.length === 0) {
return "nightly:green";
}
return "nightly:regression";
}
export function statusOf(summary, previous) {
const before = (previous ?? {}).failures ?? [];
const now = summary.failures;
const changed =
before.length !== now.length || now.some((name) => !before.includes(name));
return { verdict: verdictOf(summary), notify: changed && now.length > 0 };
}
function argument(name, fallback = "") {
const index = process.argv.indexOf(name);
if (index === -1) {
return fallback;
}
const value = process.argv[index + 1];
if (value === undefined) {
throw new Error(`${name} needs a value`);
}
return value;
}
function main() {
const out = argument("--out", "docs/ci/nightly");
const jobsPath = argument("--jobs");
if (jobsPath === "") {
throw new Error("--jobs needs the run job list");
}
const raw = JSON.parse(readFileSync(jobsPath, "utf8"));
const jobs = Array.isArray(raw) ? raw : raw.jobs;
const reports = argument("--reports");
const mutation = argument("--mutation");
const coverage = argument("--coverage");
const previousPath = argument("--previous", join(out, "latest.json"));
let previous = null;
if (existsSync(previousPath)) {
previous = JSON.parse(readFileSync(previousPath, "utf8"));
}
const summary = buildSummary({
sha: argument("--sha"),
runUrl: argument("--run-url"),
trigger: argument("--trigger", "schedule"),
finishedAt: new Date().toISOString(),
jobs,
flaky: reports === "" ? [] : collectFlaky(reports),
e2eCoverage: coverage === "" ? null : Number(coverage),
mutation: mutation === "" ? null : collectMutation(mutation),
systemCoverage: collectSystemCoverage(argument("--system-coverage")),
});
mkdirSync(resolve(out), { recursive: true });
writeFileSync(
join(out, "latest.json"),
`${JSON.stringify(summary, null, 2)}\n`,
);
writeFileSync(join(out, "report.md"), render(summary, previous));
const status = statusOf(summary, previous);
const statusPath = argument("--status");
if (statusPath !== "") {
writeFileSync(statusPath, `${JSON.stringify(status)}\n`);
}
process.stdout.write(`${status.verdict}\n`);
}
if (
process.argv[1] !== undefined &&
import.meta.url === pathToFileURL(process.argv[1]).href
) {
main();
}