scripts/spec-baseline-audit.mjs

Rama master · réplica de lectura del repositorio original.

#!/usr/bin/env node
// Re-measure every spec's expectBaselinePass and report drift.
//
// WHY THIS EXISTS SEPARATELY FROM spec-static-check.mjs
//
// The static check (anchors/shape) reads files and is ~0.3s. Baseline VALUE is a
// different animal: it requires actually running each spec's test files and comparing
// the pass count. That is expensive (all specs ~40 min) — too slow for per-PR CI, but
// exactly right as the POST-MERGE sweep that the repo already knows it needs
// (pr-gate --all-specs, SKILL.md step 5).
//
// History this exists to catch (all were invisible to `npm test`):
//   - retry-rescue-budget-split: 82 & 87 were both right alone, 88 merged (#242+#243)
//   - reasoning-continuity: 289 -> 300 (#243/#248 added tests to shared files)
//   - session-continuity-compaction-survival: 35 -> 48 (#242's tests)
//   - reasoning-dedup-incremental: 13 -> 18 (off-switch tests)
//
// Usage:
//   node scripts/spec-baseline-audit.mjs [spec-name-filter]
//
// Exit 0: every spec's baseline matches measured. Exit 1: drift found (prints the fix).
// Exit 2: at least one suite could not produce trustworthy structured evidence.
// Intended to run on master after merges (release workflow or post-merge gate).

import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { join } from 'node:path';
let acquireMutationLock, harnessEnv, materializeMutationWorkspace, parseMutationReporterOutput;
try {
  ({ acquireMutationLock, harnessEnv, materializeMutationWorkspace, parseMutationReporterOutput } =
    await import('./mutation-harness-utils.mjs'));
} catch (error) {
  console.error(`spec-baseline-audit: infrastructure unavailable: ${error.message}`);
  process.exit(2);
}

const SOURCE_ROOT = process.cwd();
const SPEC_DIR = join(SOURCE_ROOT, 'test', 'mutations');
const filter = process.argv[2];

const C = process.stdout.isTTY
  ? { red: '\x1b[31m', green: '\x1b[32m', yellow: '\x1b[33m', dim: '\x1b[2m', bold: '\x1b[1m', reset: '\x1b[0m' }
  : { red: '', green: '', yellow: '', dim: '', bold: '', reset: '' };

// Same invocation and same structured reporter as mutate-verify's runSuite. If
// this measured a count differently, a disagreement with the harness would be
// unexplainable — and the whole point is to report the TRUE number.
function measure(tests) {
  const workspace = materializeMutationWorkspace(SOURCE_ROOT);
  let stdout = '';
  let stderr = '';
  let executionFailure = null;
  try {
    stdout = execFileSync(process.execPath,
      [
        '--import', './scripts/mutation-network-deny.mjs',
        '--import', './test/setup-env.mjs',
        '--test-reporter=./scripts/mutation-harness-utils.mjs',
        '--test', ...tests,
      ],
      {
        cwd: workspace.root,
        encoding: 'utf8', env: harnessEnv(), stdio: ['ignore', 'pipe', 'pipe'],
        timeout: 15 * 60 * 1000,
        maxBuffer: 64 * 1024 * 1024,
      });
  } catch (e) {
    stdout = e.stdout || '';
    stderr = e.stderr || '';
    executionFailure = {
      status: e.status, signal: e.signal, killed: e.killed === true, code: e.code,
    };
  } finally {
    workspace.cleanup();
  }
  return parseMutationReporterOutput(stdout, stderr, executionFailure);
}

let specs;
try {
  specs = readdirSync(SPEC_DIR)
    .filter((f) => f.endsWith('.json'))
    .filter((f) => !filter || f.includes(filter))
    .sort();
} catch (error) {
  console.error(`spec-baseline-audit: cannot discover specs: ${error.message}`);
  process.exit(2);
}
if (!specs.length) {
  console.error('spec-baseline-audit: zero selected specs; evidence is untrustworthy');
  process.exit(2);
}

let drift = 0;
let untrustworthy = 0;
let total = 0;

// Lock the Owner repo once for the whole audit.  A clone-local lock would give
// every measured spec its own namespace and would not exclude another mutation
// harness running against the same checkout.
let releaseMutationLock;
try { releaseMutationLock = acquireMutationLock(SOURCE_ROOT); }
catch (err) {
  console.error(`${C.red}!${C.reset} spec-baseline-audit lock failure: ${err.message}`);
  process.exit(2);
}

try {
for (const name of specs) {
  total++;
  let spec;
  try { spec = JSON.parse(readFileSync(join(SPEC_DIR, name), 'utf8')); }
  catch (error) {
    untrustworthy++;
    console.error(`! ${name} invalid JSON: ${error.message}`);
    continue;
  }
  const tests = spec.tests;
  if (!Number.isSafeInteger(spec.expectBaselinePass) || spec.expectBaselinePass <= 0
      || !Array.isArray(tests) || !tests.length || tests.some(t => typeof t !== 'string')) {
    untrustworthy++;
    console.error(`! ${name} missing or invalid baseline/test-file evidence`);
    continue;
  }

  // Missing files invalidate evidence even when the static check was not run.
  if (tests.some((t) => !existsSync(join(process.cwd(), t)))) {
    untrustworthy++;
    console.log(`${C.red}!${C.reset} ${name}  (missing test file — evidence untrustworthy)`);
    continue;
  }

  let measured;
  try { measured = measure(tests); }
  catch (err) {
    untrustworthy++;
    console.log(`${C.red}!${C.reset} ${name}  (workspace failure: ${err.message})`);
    continue;
  }
  const { pass, fail, tests: testCount, skipped, cancelled, todo, infrastructureFailure } = measured;
  if (infrastructureFailure || testCount === 0 || testCount !== pass
      || fail !== 0 || skipped !== 0 || cancelled !== 0 || todo !== 0) {
    untrustworthy++;
    console.log(`${C.red}!${C.reset} ${name}  measurement untrustworthy `
      + `(pass=${pass} fail=${fail} tests=${testCount} skipped=${skipped} cancelled=${cancelled} todo=${todo})`);
    continue;
  }
  const ok = pass === spec.expectBaselinePass && fail === 0;
  if (ok) {
    console.log(`${C.green}✓${C.reset} ${name}  (${pass})`);
  } else {
    drift++;
    console.log(`${C.red}✗${C.reset} ${name}  expected=${spec.expectBaselinePass} measured=${pass} (fail=${fail})`);
    console.log(`     review: reproduce baseline=${pass}, inspect the changed test set, then update with provenance`);
  }
}

} finally { releaseMutationLock?.(); }
console.log(`\n${C.bold}spec-baseline-audit${C.reset}: ${total - drift - untrustworthy}/${total} match, ${drift} drifted, ${untrustworthy} untrustworthy`);
process.exit(untrustworthy ? 2 : drift ? 1 : 0);