eval-golden.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* eval-golden.mjs — golden-set eval harness for cheap-model routing (#1354)
*
* The *mechanism* in this file is design-invariant: load labeled golden cases,
* obtain each candidate model's `---SCORE_SUMMARY---` block (replayed from a
* recorded fixture for $0 deterministic CI, or live via openai-eval.mjs),
* compare it to the reference label, and exit 0/1 on an aggregate threshold.
*
* Labels are frozen reference (Claude-tier) verdicts — the metric is
* agreement-with-reference, not absolute correctness (see evals/README.md).
* ARCHETYPE agreement is the gate; SCORE is a secondary tolerance-banded signal.
*
* The remaining knobs are surfaced as named constants so they stay trivial to
* tune, with safe v1 defaults:
* - SCORE agreement → SCORE_TOLERANCE (±band; distance-to-reference)
* - CI gate threshold → MIN_ARCHETYPE_AGREEMENT
* - per-model $/run → COST_PER_RUN_USD (empty until routing rates agreed)
*
* Usage:
* node eval-golden.mjs --replay --model cheap-stub # offline, deterministic ($0)
* node eval-golden.mjs --live --model gpt-4o-mini # calls openai-eval.mjs (needs key + cv.md)
* npm run eval:golden -- --replay --model cheap-stub
*/
import { readFileSync, readdirSync, existsSync, writeFileSync, mkdtempSync, rmSync } from 'fs';
import { join, dirname } from 'path';
import { tmpdir } from 'os';
import { fileURLToPath } from 'url';
import { spawnSync } from 'child_process';
const ROOT = dirname(fileURLToPath(import.meta.url));
const GOLDEN_DIR = join(ROOT, 'evals', 'golden');
// ---------------------------------------------------------------------------
// Tunable knobs (#1354), with safe v1 defaults — see evals/README.md.
// ---------------------------------------------------------------------------
/** Max |candidate.score - label.score| still counted as agreement.
* v1: a ±band, per the distance-to-reference metric (not exact match). */
const SCORE_TOLERANCE = 0.5;
/** Fraction of cases whose ARCHETYPE must match the label for the gate to pass.
* ARCHETYPE exact-match is the clean 0/1 signal hinted at in the issue.
* v1 default: 0.8 — tunable when this is wired into the CI job. */
const MIN_ARCHETYPE_AGREEMENT = 0.8;
/** Rough $/run per model id, for the cost column. Empty until the routing
* provider rates are agreed (#1354). */
const COST_PER_RUN_USD = {};
// ---------------------------------------------------------------------------
// CLI args
// ---------------------------------------------------------------------------
const args = process.argv.slice(2);
if (args.includes('--help') || args.includes('-h')) {
console.log(`eval-golden.mjs — golden-set eval harness (#1354)
--replay Replay recorded fixtures (default; offline, $0, deterministic)
--live Call the model live via openai-eval.mjs (needs key + cv.md)
--model <id> Candidate model id to evaluate (default: cheap-stub)
--golden <dir> Golden-set directory (default: evals/golden)
--fixtures <dir> Replay fixtures directory (default: sibling of --golden)
--help Show this help
`);
process.exit(0);
}
const mode = args.includes('--live') ? 'live' : 'replay';
const model = argValue('--model') || 'cheap-stub';
const goldenDir = argValue('--golden') || GOLDEN_DIR;
// Keep fixtures next to the golden set so a custom --golden dir resolves its
// own fixtures (the default lands on evals/fixtures); override with --fixtures.
const fixtureDir = argValue('--fixtures') || join(dirname(goldenDir), 'fixtures');
/**
* Read the value following a `--flag` token in argv.
*
* @param {string} flag - The flag whose following value to return.
* @returns {string|undefined} The value, or undefined if the flag is absent/last.
*/
function argValue(flag) {
const i = args.indexOf(flag);
return i >= 0 && i + 1 < args.length ? args[i + 1] : undefined;
}
/**
* Flatten a model id into a filesystem-safe fixture token.
*
* Provider ids like "deepseek/deepseek-chat" contain slashes that would be read
* as path separators; collapse any non-[A-Za-z0-9._-] run to a single "-" so the
* fixture stays a single flat file.
*
* @param {string} m - The candidate model id.
* @returns {string} A path-safe token for the fixture filename.
*/
function fixtureModelId(m) {
return m.replace(/[^A-Za-z0-9._-]+/g, '-');
}
// ---------------------------------------------------------------------------
// Shared SCORE_SUMMARY parser — same contract every *-eval.mjs already emits.
// ---------------------------------------------------------------------------
/**
* Parse the machine-readable summary block produced by the eval scripts.
*
* @param {string} text - Raw model output containing a SCORE_SUMMARY block.
* @returns {{score: number, archetype: string}} Parsed score/archetype; score
* is NaN and archetype is "unknown" when the block is missing or malformed.
*/
function parseSummary(text) {
const block = text.match(/---SCORE_SUMMARY---\s*([\s\S]*?)---END_SUMMARY---/);
const field = (key) => {
const m = block && block[1].match(new RegExp(`${key}:\\s*(.+)`));
return m ? m[1].trim() : '';
};
return {
score: parseFloat(field('SCORE')),
archetype: (field('ARCHETYPE') || 'unknown').toLowerCase(),
};
}
// ---------------------------------------------------------------------------
// Obtain one candidate completion (replay fixture or live openai-eval call).
// ---------------------------------------------------------------------------
/**
* Return the candidate model's raw evaluation text for one golden case.
*
* In replay mode this reads a recorded fixture so the gate is offline and
* deterministic; in live mode it shells out to openai-eval.mjs, reusing the
* real prompt-assembly path rather than duplicating it here.
*
* @param {{id: string, jd: string}} testCase - The golden case being run.
* @returns {string} Raw model output (expected to contain a SCORE_SUMMARY block).
*/
function getCompletion(testCase) {
if (mode === 'replay') {
// Slash-form provider ids (e.g. "deepseek/deepseek-chat") must not become
// path separators, or the fixture lands in a phantom subdirectory. Sanitize
// to a flat filename — record fixtures under the same sanitized name.
const fixture = join(fixtureDir, `${testCase.id}__${fixtureModelId(model)}.txt`);
if (!existsSync(fixture)) {
throw new Error(`missing replay fixture: ${fixture} — record it or run --live`);
}
return readFileSync(fixture, 'utf8');
}
// live: write the JD to a temp file and run the existing evaluator.
const dir = mkdtempSync(join(tmpdir(), 'eval-golden-'));
try {
const jdFile = join(dir, 'jd.txt');
writeFileSync(jdFile, testCase.jd);
const res = spawnSync(process.execPath,
[join(ROOT, 'openai-eval.mjs'), '--file', jdFile, '--model', model, '--no-save'],
{ encoding: 'utf8', env: process.env, timeout: 360000 });
if (res.status !== 0) {
throw new Error(`openai-eval.mjs exited ${res.status}: ${(res.stderr || '').slice(0, 200)}`);
}
return res.stdout || '';
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
/**
* Median of a numeric array (0 for an empty array).
*
* @param {number[]} xs - Values to summarize.
* @returns {number} The median value.
*/
function median(xs) {
if (xs.length === 0) return 0;
const s = [...xs].sort((a, b) => a - b);
const mid = Math.floor(s.length / 2);
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
}
// ---------------------------------------------------------------------------
// Run
// ---------------------------------------------------------------------------
if (!existsSync(goldenDir)) {
console.error(`❌ golden-set directory not found: ${goldenDir}`);
process.exit(1);
}
let cases;
try {
cases = readdirSync(goldenDir)
.filter((f) => f.endsWith('.json'))
.map((f) => {
const parsed = JSON.parse(readFileSync(join(goldenDir, f), 'utf8'));
if (typeof parsed?.id !== 'string' ||
typeof parsed?.jd !== 'string' ||
typeof parsed?.label?.archetype !== 'string' ||
typeof parsed?.label?.score !== 'number') {
throw new Error(`invalid golden case ${f}: need string id/jd and label.{archetype:string, score:number}`);
}
return parsed;
});
} catch (err) {
console.error(`❌ ${err.message || err}`);
process.exit(1);
}
if (cases.length === 0) {
console.error(`❌ no golden cases (*.json) in ${goldenDir}`);
process.exit(1);
}
console.log(`\ngolden-set eval — model "${model}" (${mode}), ${cases.length} case(s)`);
console.log(`(row ✅ needs both archetype + score; the gate counts archetype agreement only)\n`);
let archetypeHits = 0;
const deltas = [];
const latencies = [];
for (const tc of cases) {
const t0 = Date.now();
let parsed;
try {
parsed = parseSummary(getCompletion(tc));
} catch (err) {
console.log(` ❌ ${tc.id}: ${err.message}`);
deltas.push(NaN);
continue;
}
const latencyMs = Date.now() - t0;
latencies.push(latencyMs);
const archetypeMatch = parsed.archetype === String(tc.label.archetype).toLowerCase();
const delta = Math.abs(parsed.score - tc.label.score);
const scoreOk = Number.isFinite(delta) && delta <= SCORE_TOLERANCE;
if (archetypeMatch) archetypeHits++;
deltas.push(delta);
const ok = archetypeMatch && scoreOk;
console.log(
` ${ok ? '✅' : '❌'} ${tc.id}: ` +
`archetype ${parsed.archetype} vs ${String(tc.label.archetype).toLowerCase()} ` +
`(${archetypeMatch ? 'match' : 'MISS'}); ` +
`score ${parsed.score} vs ${tc.label.score} (Δ${Number.isFinite(delta) ? delta.toFixed(2) : 'n/a'}); ` +
`${mode === 'live' ? `${latencyMs}ms` : 'replay'}`,
);
}
const agreement = archetypeHits / cases.length;
const finiteDeltas = deltas.filter(Number.isFinite);
const meanDelta = finiteDeltas.length ? finiteDeltas.reduce((a, b) => a + b, 0) / finiteDeltas.length : NaN;
// Cases whose SCORE was missing/malformed produce a NaN delta and drop out of
// the mean — surface that count so a model can't hide failures behind a low mean.
const unscored = cases.length - finiteDeltas.length;
const cost = COST_PER_RUN_USD[model];
console.log('\n ── summary ──');
console.log(` archetype agreement : ${(agreement * 100).toFixed(0)}% (gate ≥ ${(MIN_ARCHETYPE_AGREEMENT * 100).toFixed(0)}%)`);
console.log(` mean |Δscore| : ${Number.isFinite(meanDelta) ? meanDelta.toFixed(2) : 'n/a'} over ${finiteDeltas.length}/${cases.length} scored${unscored ? ` (${unscored} unscored)` : ''} (tolerance ±${SCORE_TOLERANCE})`);
if (mode === 'live') console.log(` median latency : ${median(latencies)}ms`);
console.log(` est. $/run : ${cost != null ? `$${cost}` : 'n/a — TODO(#1354)'}`);
const passed = agreement >= MIN_ARCHETYPE_AGREEMENT;
console.log(`\n ${passed ? '✅ PASS' : '❌ FAIL'} — archetype agreement ${passed ? 'meets' : 'below'} gate\n`);
process.exit(passed ? 0 : 1);