docker/context-profiles/ai-eval-lib.js
Rama main · réplica de lectura del repositorio original.
'use strict';
// Development-only evaluator. It lives under docker/ so the npm package never ships it.
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const { isDeepStrictEqual } = require('node:util');
const LIB = path.join(__dirname, '../../scripts/lib');
const { loadContextRegistry } = require(path.join(LIB, 'context-pack-registry'));
const { compileContextProfile } = require(path.join(LIB, 'context-profiles'));
const { resolveTaskContext, resolveDeclinedFallback } = require(path.join(LIB, 'context-selection'));
const { proposeTaskContext } = require(path.join(LIB, 'context-profile-proposal'));
const { resolveExecutable, fingerprintExecutable } = require(path.join(LIB, 'context-profile-native-executable'));
const { launchTaskContext } = require(path.join(LIB, 'context-profile-launch'));
const { applyStore } = require(path.join(LIB, 'context-profile-store'));
const { prepareNativeProfile, getNativeProfileStatus } = require(path.join(LIB, 'context-profile-native'));
const { DEFAULT_REPO_ROOT, digestObject, createSourceReader } = require(path.join(LIB, 'context-profile-support'));
const io = require(path.join(LIB, 'context-profile-store-fs'));
const ARMS = Object.freeze(['full', 'manual-lean', 'auto-lean', 'ecc-legacy', 'baseline']);
const CORPUS_PATH = path.join(__dirname, 'ai-corpus.json');
const LEGACY_PIN_PATH = path.join(__dirname, 'legacy-source.json');
const CHECK_FILE = '.ecc-eval-check.cjs';
const IMPLEMENTATION = ['docker/context-profiles/ai-eval-lib.js', 'docker/context-profiles/ai-eval.js',
'docker/context-profiles/legacy-source.json',
'manifests/context-packs/skill-triggers@1.json',
'scripts/lib/context-profile-launch.js', 'scripts/lib/context-selection.js',
'scripts/lib/context-retrieval.js',
'scripts/lib/context-profile-proposal.js', 'scripts/lib/context-profiles.js',
'scripts/lib/context-profile-support.js', 'scripts/lib/context-pack-registry.js',
'scripts/lib/context-profile-native-executable.js', 'scripts/lib/context-profile-native.js',
'scripts/lib/context-profile-store.js', 'scripts/lib/context-profile-store-fs.js'];
const BLOCKS = Object.freeze({ excluded: /Context ID is excluded:/,
'native-authority': /requires native authority or dynamic-content review/,
'manual-only': /Context ID is manual-only:/, 'opt-out-conflict': /noWorkflow conflicts/, 'unknown-id': /Unknown context ID:/ });
const ENV_KEYS = ['PATH', 'HOME', 'USERPROFILE', 'CODEX_HOME', 'TMPDIR', 'LANG', 'SystemRoot'];
const CLAUDE_ENV_KEYS = ['PATH', 'HOME', 'USERPROFILE', 'CLAUDE_CONFIG_DIR', 'TMPDIR', 'LANG', 'SystemRoot'];
const bounded = (value, min, max) => Number.isSafeInteger(value) && value >= min && value <= max;
const exists = file => Boolean(fs.lstatSync(file, { throwIfNoEntry: false }));
function loadCorpus(file = CORPUS_PATH) { return JSON.parse(fs.readFileSync(file, 'utf8')); }
function safeRelative(file) {
return typeof file === 'string' && file.length > 0 && file.length <= 200 && !path.isAbsolute(file)
&& !file.startsWith('.') && !file.includes('\\') && file.split('/').every(part => part && part !== '..' && part !== '.');
}
function validateCorpus(corpus) {
if (corpus?.schemaVersion === 'ecc.context-eval-complex-corpus.v1') return validateComplexCorpus(corpus);
if (corpus?.schemaVersion !== 'ecc.context-eval-corpus.v2'
|| !Array.isArray(corpus.selection) || !Array.isArray(corpus.tasks)
|| !bounded(corpus.selection.length, 1, 200) || !bounded(corpus.tasks.length, 1, 200)
|| corpus.minimumDistinctTasks !== 30 || corpus.nonInferiorityMargin !== 0.05) {
throw new Error('Invalid preregistered corpus');
}
for (const cases of [corpus.selection, corpus.tasks]) validateCorpusIds(cases);
for (const task of corpus.tasks) {
const files = Object.entries(task.files || {});
if (!Array.isArray(task.manualIds) || task.manualIds.length > 1 || !bounded(files.length, 1, 8)
|| files.some(([file, content]) => !safeRelative(file) || typeof content !== 'string' || Buffer.byteLength(content) > 16384)
|| typeof task.check !== 'string' || !bounded(Buffer.byteLength(task.check), 1, 16384)) {
throw new Error('Invalid corpus task');
}
}
}
function validateCorpusIds(cases) {
if (new Set(cases.map(c => c.id)).size !== cases.length) throw new Error('Duplicate corpus ID');
for (const item of cases) {
if (!/^[a-z][a-z0-9-]{0,63}$/.test(item.id) || typeof item.query !== 'string'
|| !bounded(Buffer.byteLength(item.query), 1, 8192)) throw new Error('Invalid corpus case');
}
}
// Complex corpora hold a few realistic multi-file tasks with scored hidden graders. Sample gates
// are descriptive at this size, so the distinct-task minimum relaxes to the corpus itself.
function invalidComplexStep(step) {
return typeof step.query !== 'string' || !bounded(Buffer.byteLength(step.query), 1, 8192)
|| typeof step.check !== 'string' || !bounded(Buffer.byteLength(step.check), 1, 65536)
|| (step.checkTimeoutMs !== undefined && !bounded(step.checkTimeoutMs, 1, 120000))
|| (step.manualIds !== undefined && (!Array.isArray(step.manualIds) || step.manualIds.length > 3));
}
function validateComplexTaskFiles(task) {
const files = Object.entries(task.files || {});
if (!Array.isArray(task.manualIds) || task.manualIds.length > 3 || !bounded(files.length, 1, 24)
|| files.some(([file, content]) => !safeRelative(file) || typeof content !== 'string' || Buffer.byteLength(content) > 65536)) {
throw new Error('Invalid corpus task');
}
}
function validateComplexTask(task) {
if (!/^[a-z][a-z0-9-]{0,63}$/.test(task.id)) throw new Error('Invalid corpus case');
if (task.steps === undefined
&& (typeof task.query !== 'string' || !bounded(Buffer.byteLength(task.query), 1, 8192))) throw new Error('Invalid corpus case');
validateComplexTaskFiles(task);
if (task.steps !== undefined) {
// Stepped (chained) task: sequential tickets graded in one accumulating workspace.
if (!Array.isArray(task.steps) || !bounded(task.steps.length, 2, 8) || task.steps.some(invalidComplexStep)) {
throw new Error('Invalid corpus task');
}
} else if (typeof task.check !== 'string' || !bounded(Buffer.byteLength(task.check), 1, 65536)
|| (task.checkTimeoutMs !== undefined && !bounded(task.checkTimeoutMs, 1, 120000))) {
throw new Error('Invalid corpus task');
}
}
function validateComplexCorpus(corpus) {
if (!Array.isArray(corpus.selection) || !Array.isArray(corpus.tasks)
|| !bounded(corpus.selection.length, 0, 50) || !bounded(corpus.tasks.length, 1, 10)
|| corpus.minimumDistinctTasks !== corpus.tasks.length || corpus.nonInferiorityMargin !== 0.05) {
throw new Error('Invalid preregistered corpus');
}
validateCorpusIds(corpus.selection);
if (new Set(corpus.tasks.map(c => c.id)).size !== corpus.tasks.length) throw new Error('Duplicate corpus ID');
for (const task of corpus.tasks) validateComplexTask(task);
}
function sourceSnapshot(repoRoot) {
const registry = loadContextRegistry({ repoRoot });
const profiles = ['full@1', 'lean@1'].map(profileId => compileContextProfile({ repoRoot, profileId }));
// Implementation modules are loaded from this evaluator's checkout; repoRoot may be a fixture registry.
const reader = createSourceReader(DEFAULT_REPO_ROOT);
const implementation = IMPLEMENTATION.map(file => ({ path: file, digest: reader.read(file).digest }));
const packageJson = JSON.parse(reader.read('package.json').content.toString('utf8'));
const runtime = { node: process.versions.node, dependencies: {
ajv: packageJson.dependencies.ajv, 'js-yaml': packageJson.dependencies['js-yaml'] } };
return { registry, profiles, sourceDigest: digestObject({ registryDigest: registry.registryDigest,
planDigests: profiles.map(p => p.planDigest), implementation, runtime }), runtime };
}
const EFFORTS = ['low', 'medium', 'high', 'xhigh', 'max', 'ultra'];
function providerFamily(executable) {
const base = path.basename(String(executable || '')).toLowerCase();
if (base.includes('claude')) return 'claude';
if (base.includes('codex')) return 'codex';
throw new Error('Provider executable must name a Claude or Codex CLI');
}
function resolveFamily(provider, executable) {
if (provider !== undefined && provider !== null) {
if (!['claude', 'codex'].includes(provider)) throw new Error('Provider must be claude or codex');
return provider;
}
if (executable) return providerFamily(executable);
return 'codex';
}
function providerPin(model, executable, effort) {
if (model === undefined && executable === undefined && effort === undefined) return null;
if (typeof model !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,99}$/.test(model)
|| !path.isAbsolute(executable || '')) throw new Error('Provider pin requires model and absolute executable');
if (effort !== undefined && !EFFORTS.includes(effort)) throw new Error('Invalid reasoning effort');
return { modelDigest: digestObject(model), executableDigest: resolveExecutable(executable).digest,
...(effort === undefined ? {} : { effort }) };
}
function preregister({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), repeats = 1, model, executable, effort, arms } = {}) {
validateCorpus(corpus);
if (!bounded(repeats, 1, 20)) throw new Error('Invalid repeat count');
const armList = arms === undefined ? [...ARMS] : arms;
if (!Array.isArray(armList) || !armList.length || new Set(armList).size !== armList.length
|| armList.some(arm => !ARMS.includes(arm))) throw new Error('Invalid arm subset');
const source = sourceSnapshot(repoRoot);
const value = { schemaVersion: 'ecc.context-eval-registration.v2', corpusDigest: digestObject(corpus),
sourceDigest: source.sourceDigest, registryDigest: source.registry.registryDigest,
providerPin: providerPin(model, executable, effort), runtime: source.runtime,
arms: armList, repeats, minimumDistinctTasks: corpus.minimumDistinctTasks, nonInferiorityMargin: 0.05,
confidence: 0.95, sampling: 'fixed-purposive-pilot',
design: corpus.schemaVersion === 'ecc.context-eval-complex-corpus.v1'
? 'paired-native-installs-hidden-scored-complex-tasks'
: 'paired-native-installs-hidden-graded-coding-tasks',
order: corpus.tasks.flatMap((task, index) => Array.from({ length: repeats }, (_, repeat) => ({
id: task.id, repeat, arms: armList.map((_, offset) => armList[(index + repeat + offset) % armList.length]),
}))), selectionIds: corpus.selection.map(c => c.id) };
return { ...value, registrationDigest: digestObject(value) };
}
// Parse in memory only. No event objects, paths, provider messages or error text enter reports.
function parseCodexJsonl(stdout) {
const invalid = { valid: false, text: '', usage: null };
if (typeof stdout !== 'string' || Buffer.byteLength(stdout) > 1024 * 1024) return invalid;
let text = '';
let completions = 0;
let usage = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 };
try {
for (const line of stdout.split('\n').filter(line => line.trim())) {
const event = JSON.parse(line);
if (!event || typeof event !== 'object' || ['error', 'turn.failed'].includes(event.type)) return invalid;
if (event.type === 'item.completed' && event.item?.type === 'agent_message') {
if (typeof event.item.text !== 'string') return invalid;
text = event.item.text;
}
if (event.type !== 'turn.completed') continue;
const u = event.usage;
if (!u || ![u.input_tokens, u.cached_input_tokens, u.output_tokens].every(v => bounded(v, 0, 1e9))
|| u.cached_input_tokens > u.input_tokens) return invalid;
completions++;
usage = { inputTokens: usage.inputTokens + u.input_tokens,
cachedInputTokens: usage.cachedInputTokens + u.cached_input_tokens,
outputTokens: usage.outputTokens + u.output_tokens };
}
} catch { return invalid; }
return completions === 1 ? { valid: true, text, usage } : invalid;
}
// Claude print-mode emits exactly one result JSON object. Fresh input folds cache creations;
// cache reads are reported separately. is_error results are provider failures, not parse failures.
function parseClaudeJson(stdout) {
const invalid = { valid: false, text: '', usage: null };
if (typeof stdout !== 'string' || Buffer.byteLength(stdout) > 1024 * 1024) return invalid;
let result = null;
let results = 0;
try {
for (const line of stdout.split('\n').filter(line => line.trim())) {
const event = JSON.parse(line);
if (!event || typeof event !== 'object' || Array.isArray(event)) return invalid;
if (event.type !== 'result') continue;
results++;
result = event;
}
} catch { return invalid; }
if (results !== 1) return invalid;
if (result.is_error !== false || typeof result.result !== 'string') return { ...invalid, error: true };
const u = result.usage;
if (!u || ![u.input_tokens, u.cache_creation_input_tokens, u.cache_read_input_tokens, u.output_tokens]
.every(value => bounded(value, 0, 1e9))) return { ...invalid, error: true };
return { valid: true, text: result.result,
usage: { inputTokens: u.input_tokens + u.cache_creation_input_tokens,
cachedInputTokens: u.cache_read_input_tokens, outputTokens: u.output_tokens } };
}
function privateEntry(file, directory) {
const stat = fs.lstatSync(file, { throwIfNoEntry: false });
return Boolean(stat) && !stat.isSymbolicLink() && (directory ? stat.isDirectory() : stat.isFile())
&& (process.platform === 'win32' || ((stat.mode & 0o077) === 0 && (!process.getuid || stat.uid === process.getuid())));
}
/**
* Subscription credentials stay in a dedicated evaluator login home. Each call leases auth.json into the
* isolated CODEX_HOME, returns refreshed tokens afterwards and always removes the leased copy.
*/
function createAuthLease(authHome) {
if (typeof authHome !== 'string' || !path.isAbsolute(authHome)) throw new Error('Auth home must be an absolute path');
const real = fs.realpathSync(authHome);
const forbidden = [path.join(os.homedir(), '.codex'), process.env.CODEX_HOME].filter(Boolean)
.map(file => (exists(file) ? fs.realpathSync(file) : path.resolve(file)));
if (forbidden.includes(real)) throw new Error('Auth home must be a dedicated evaluator login home, not your Codex home');
const source = path.join(real, 'auth.json');
if (!privateEntry(real, true) || !privateEntry(source, false)) {
throw new Error('Auth home must be a private directory containing a private auth.json; see the evaluation guide');
}
return {
mode: 'subscription-lease',
run(codexHome, work) {
const leased = path.join(codexHome, 'auth.json');
const original = fs.readFileSync(source);
fs.writeFileSync(leased, original, { flag: 'wx', mode: 0o600 });
try { return work(); } finally {
try {
const after = fs.readFileSync(leased);
if (!after.equals(original)) {
JSON.parse(after.toString('utf8'));
const temp = `${source}.${process.pid}.tmp`;
try {
fs.writeFileSync(temp, after, { flag: 'wx', mode: 0o600 });
fs.renameSync(temp, source);
} finally { fs.rmSync(temp, { force: true }); }
}
} catch { /* An unreadable refresh keeps the previous login; the next call reports any auth failure. */ }
fs.rmSync(leased, { force: true });
}
},
};
}
/**
* Claude subscription logins live in the macOS Keychain as a JSON wrapper. The lease reads the
* current access token per call into the child environment only; it is never persisted or reported.
*/
function readClaudeKeychainToken() {
if (process.platform !== 'darwin') throw new Error('Claude Keychain login requires macOS; provide CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY');
const result = spawnSync('security', ['find-generic-password', '-s', 'Claude Code-credentials', '-w'],
{ encoding: 'utf8', shell: false, timeout: 15000, killSignal: 'SIGKILL', maxBuffer: 65536 });
if (result.status !== 0 || result.error) throw new Error('Claude Keychain login is unavailable; provide CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY');
let parsed;
try { parsed = JSON.parse(result.stdout); }
catch { throw new Error('Claude Keychain login is unreadable; provide CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY'); }
const token = parsed?.claudeAiOauth?.accessToken;
if (typeof token !== 'string' || !token) throw new Error('Claude Keychain login is unrecognized; provide CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY');
return token;
}
function createClaudeProvider({ allowRealProvider = false, allowCredentialedTools = false, executable, model,
apiKey = process.env.ANTHROPIC_API_KEY, oauthToken = process.env.CLAUDE_CODE_OAUTH_TOKEN,
tokenSource = readClaudeKeychainToken, persistSessions = false, execute = spawnSync } = {}) {
if (allowRealProvider !== true) throw new Error('Real provider requires explicit opt-in');
if (!model || !executable) throw new Error('Real provider requires a model and absolute executable');
let lease = null;
let authentication;
if (oauthToken) authentication = 'oauth-env';
else if (apiKey) authentication = 'api-key';
else if (typeof tokenSource === 'function') {
lease = { mode: 'subscription-keychain-lease',
run(env, work) { env.CLAUDE_CODE_OAUTH_TOKEN = tokenSource(); return work(); } };
authentication = lease.mode;
} else throw new Error('Real provider requires CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY, or the Claude Keychain login');
const pin = providerPin(model, executable, undefined);
const binary = resolveExecutable(executable);
const provider = request => {
if (fingerprintExecutable(binary.path).digest !== pin.executableDigest) fail('source-drift');
const selection = request.phase === 'selection';
if (!selection && !allowCredentialedTools) {
throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required');
}
// Selection is tool-free and read-only; task execution may edit and run commands in the workspace.
// Claude has no cwd-write sandbox flag, so containment relies on the isolated home and temp workspace.
const args = ['--print', '--output-format', 'json',
...(persistSessions ? [] : ['--no-session-persistence']),
...(selection ? ['--tools', ''] : ['--permission-mode', 'bypassPermissions']),
'--model', model];
const env = Object.fromEntries(CLAUDE_ENV_KEYS.filter(key => typeof request.env?.[key] === 'string')
.map(key => [key, request.env[key]]));
env.DISABLE_NON_ESSENTIAL_MODEL_CALLS = '1';
if (authentication === 'oauth-env') env.CLAUDE_CODE_OAUTH_TOKEN = oauthToken;
if (authentication === 'api-key') env.ANTHROPIC_API_KEY = apiKey;
const call = () => execute(binary.path, args, { input: request.input, cwd: request.cwd, env,
encoding: 'utf8', shell: false, timeout: request.timeoutMs, killSignal: 'SIGKILL',
maxBuffer: request.maxBuffer });
return lease ? lease.run(env, call) : call();
};
provider.authentication = authentication;
return provider;
}
function createCodexProvider({ allowRealProvider = false, executable, model, effort, authHome,
apiKey = process.env.CODEX_API_KEY, execute = spawnSync } = {}) {
if (allowRealProvider !== true) throw new Error('Real provider requires explicit opt-in');
if (!model || !executable) throw new Error('Real provider requires a model and absolute executable');
if (!authHome && !apiKey) throw new Error('Real provider requires --auth-home (subscription login) or CODEX_API_KEY');
const lease = authHome ? createAuthLease(authHome) : null;
const pin = providerPin(model, executable, effort);
const binary = resolveExecutable(executable);
const provider = request => {
if (fingerprintExecutable(binary.path).digest !== pin.executableDigest) fail('source-drift');
const args = ['exec', '--json', '--ephemeral', '--skip-git-repo-check',
'--sandbox', request.phase === 'selection' ? 'read-only' : 'workspace-write',
// Connected ChatGPT apps and account plugin installs stay out of every arm.
'--disable', 'apps', '--disable', 'remote_plugin',
'-c', 'approval_policy="never"', ...(effort ? ['-c', `model_reasoning_effort="${effort}"`] : []),
'--model', model, '-'];
const env = Object.fromEntries(ENV_KEYS.filter(key => typeof request.env?.[key] === 'string')
.map(key => [key, request.env[key]]));
if (!lease) env.CODEX_API_KEY = apiKey;
const call = () => execute(binary.path, args, { input: request.input, cwd: request.cwd, env,
encoding: 'utf8', shell: false, timeout: request.timeoutMs, killSignal: 'SIGKILL',
maxBuffer: request.maxBuffer });
return lease ? lease.run(env.CODEX_HOME, call) : call();
};
provider.authentication = lease ? lease.mode : 'api-key';
return provider;
}
/** Real Lean and Full installs, prepared through the same isolated native adapter users get. */
function prepareEnvironments({ repoRoot, executable, root }) {
const binary = resolveExecutable(executable);
const environments = {};
for (const [name, profileId, selectionMode] of [['full', 'full@1', 'manual'], ['lean', 'lean@1', 'auto']]) {
const options = { stateRoot: path.join(root, name, 'managed'), nativeRoot: path.join(root, name, 'native') };
fs.mkdirSync(path.join(root, name), { mode: 0o700 });
applyStore({ repoRoot, stateRoot: options.stateRoot, target: 'codex', selectionMode, profileId });
const status = prepareNativeProfile({ ...options, codexPath: executable });
if (!status.ready) throw new Error(`Native ${name} install is not ready`);
// A signed-in Codex records task-directory trust in config.toml and downloads account-provided
// plugins into plugins/. Restoring the prepared state after every call keeps trials identical;
// any other change still fails verification as drift.
const config = path.join(status.codexHome, 'config.toml');
const prepared = fs.readFileSync(config);
const plugins = path.join(status.codexHome, 'plugins');
const listing = directory => (exists(directory) ? fs.readdirSync(directory) : []);
const preparedPlugins = new Set(listing(plugins));
const preparedCache = new Set(listing(path.join(plugins, 'cache')));
environments[name] = { profileId, skills: status.selectedIds.length,
launch: { home: status.home, codexHome: status.codexHome, codexPath: status.codexPath,
executableDigest: status.executableDigest },
restore() {
fs.writeFileSync(config, prepared);
for (const entry of listing(plugins)) if (!preparedPlugins.has(entry)) fs.rmSync(path.join(plugins, entry), { recursive: true, force: true });
for (const entry of listing(path.join(plugins, 'cache'))) {
if (!preparedCache.has(entry)) fs.rmSync(path.join(plugins, 'cache', entry), { recursive: true, force: true });
}
},
verify() {
let ready = false;
try { ready = getNativeProfileStatus(options).ready; } catch { ready = false; }
if (!ready) fail('environment-drift');
} };
}
// Baseline arm: an empty native home with no ECC install, for provider-overhead subtraction.
const home = path.join(root, 'baseline', 'home');
fs.mkdirSync(path.join(home, '.codex'), { recursive: true, mode: 0o700 });
environments.baseline = { profileId: null, skills: 0, restore() {},
launch: { home, codexHome: path.join(home, '.codex'), codexPath: binary.path, executableDigest: binary.digest },
verify() { if (fingerprintExecutable(binary.path).digest !== binary.digest) fail('environment-drift'); } };
return environments;
}
function installClaudeSkills({ payload, home }) {
const config = path.join(home, '.claude');
const installed = path.join(config, 'skills');
fs.mkdirSync(installed, { recursive: true, mode: 0o700 });
for (const entry of fs.readdirSync(payload)) {
fs.cpSync(path.join(payload, entry), path.join(installed, entry), { recursive: true, errorOnExist: true, force: false });
}
return { config, installed };
}
function claudeEnvironment({ name, binary, home, config, installed, profileId, skills, sourceSha = null }) {
const managed = () => digestObject(io.inventory(installed));
const prepared = managed();
return [name, { profileId, skills, sourceSha,
launch: { home, claudeConfigDir: config, claudePath: binary.path, executableDigest: binary.digest },
restore() {},
verify() {
if (fingerprintExecutable(binary.path).digest !== binary.digest) fail('environment-drift');
let observed = null;
try { observed = managed(); } catch { observed = null; }
if (observed !== prepared) fail('environment-drift');
} }];
}
/** The pre-scoping ECC source, pinned by commit so the ecc-legacy arm is reproducible. */
function exportLegacySource({ repoRoot = DEFAULT_REPO_ROOT, destination,
pin = JSON.parse(fs.readFileSync(LEGACY_PIN_PATH, 'utf8')) } = {}) {
if (!/^[a-f0-9]{40}$/.test(pin?.sha || '')) throw new Error('Invalid legacy source pin');
if (!path.isAbsolute(destination || '')) throw new Error('Legacy destination must be absolute');
const resolved = spawnSync('git', ['-C', repoRoot, 'rev-parse', '--verify', `${pin.sha}^{commit}`],
{ encoding: 'utf8', shell: false, timeout: 30000, killSignal: 'SIGKILL' });
if (resolved.status !== 0 || resolved.error || resolved.stdout.trim() !== pin.sha) {
throw new Error('Legacy source pin is unavailable in this repository');
}
fs.mkdirSync(destination, { recursive: true, mode: 0o700 });
const tar = path.join(destination, 'legacy.tar');
const archive = spawnSync('git', ['-C', repoRoot, 'archive', '--format=tar', '-o', tar, pin.sha, 'skills'],
{ encoding: 'utf8', shell: false, timeout: 60000, killSignal: 'SIGKILL' });
const extract = archive.status === 0 && !archive.error
? spawnSync('tar', ['-xf', tar, '-C', destination], { encoding: 'utf8', shell: false, timeout: 60000, killSignal: 'SIGKILL' })
: archive;
fs.rmSync(tar, { force: true });
const payload = path.join(destination, 'skills');
if (extract.status !== 0 || extract.error || !exists(payload) || !fs.readdirSync(payload).length) {
throw new Error('Legacy source export failed');
}
return { root: destination, sha: pin.sha };
}
/** Real Claude installs in isolated config homes. Managed-skill drift aborts; there is no
* provider bookkeeping to restore because isolated Claude runs do not mutate the managed tree. */
function prepareClaudeEnvironments({ repoRoot, executable, root, legacySource = null }) {
const binary = resolveExecutable(executable);
const environments = {};
for (const [name, profileId, selectionMode] of [['full', 'full@1', 'manual'], ['lean', 'lean@1', 'auto']]) {
const stateRoot = path.join(root, name, 'managed');
fs.mkdirSync(path.join(root, name), { mode: 0o700 });
const status = applyStore({ repoRoot, stateRoot, target: 'claude', selectionMode, profileId });
const home = path.join(root, name, 'home');
const { config, installed } = installClaudeSkills({ payload: path.join(status.generationRoot, 'skills'), home });
const [key, env] = claudeEnvironment({ name, binary, home, config, installed, profileId, skills: status.selectedIds.length });
environments[key] = env;
}
if (legacySource) {
// ecc-legacy: the typical pre-scoping install — the full skill library from the pinned
// pre-ECC-029 commit, launched bare with no ECC context block.
const home = path.join(root, 'ecc-legacy', 'home');
const { config, installed } = installClaudeSkills({ payload: path.join(legacySource.root, 'skills'), home });
const [key, env] = claudeEnvironment({ name: 'ecc-legacy', binary, home, config, installed,
profileId: null, skills: fs.readdirSync(installed).length, sourceSha: legacySource.sha });
environments[key] = env;
}
// Baseline arm: an empty config home with no ECC install, for provider-overhead subtraction.
const baselineHome = path.join(root, 'baseline', 'home');
const baselineConfig = path.join(baselineHome, '.claude');
fs.mkdirSync(baselineConfig, { recursive: true, mode: 0o700 });
environments.baseline = { profileId: null, skills: 0, sourceSha: null, restore() {},
launch: { home: baselineHome, claudeConfigDir: baselineConfig, claudePath: binary.path, executableDigest: binary.digest },
verify() { if (fingerprintExecutable(binary.path).digest !== binary.digest) fail('environment-drift'); } };
return environments;
}
function syntheticEnvironments(root) {
const executable = resolveExecutable(process.execPath);
return Object.fromEntries(['full', 'lean', 'ecc-legacy', 'baseline'].map(name => {
const home = path.join(root, name, 'home');
fs.mkdirSync(path.join(home, '.codex'), { recursive: true, mode: 0o700 });
return [name, { profileId: ['baseline', 'ecc-legacy'].includes(name) ? null : `${name}@1`, skills: null, sourceSha: null,
verify() {}, restore() {},
launch: { home, codexHome: path.join(home, '.codex'), codexPath: executable.path, executableDigest: executable.digest } }];
}));
}
function checkArguments(cwd, file = CHECK_FILE, writable = false) {
const major = Number(process.versions.node.split('.')[0]);
const flag = major >= 22 ? '--permission' : major >= 20 ? '--experimental-permission' : null;
// A directory grant covers its children. Node 20.20.2 can abort in its native
// permission radix tree when the same directory is also granted as "cwd/*".
return flag ? [flag, `--allow-fs-read=${cwd}`,
// Stepped graders exercise stateful apps (persistence); single-step graders stay read-only.
...(writable ? [`--allow-fs-write=${cwd}`] : []), file] : [file];
}
// The hidden grader enters the workspace only after the agent exits, and runs read-only where Node supports it.
// A grader may print one `ECC_EVAL_SCORE {"score":0..1}` line for partial credit; without it the exit
// status alone decides (exit 0 scores 1). Outcome success still requires a full score. Stepped tasks
// grade each step with a distinct grader file so earlier graders stay readable in the workspace.
const SCORE_LINE = /^\s*ECC_EVAL_SCORE\s+(\{[^\n]*\})\s*$/m;
function runScoredCheck(cwd, source, timeoutMs = 10000, step = null) {
const name = step === null ? CHECK_FILE : `.ecc-eval-check-${step}.cjs`;
const file = path.join(cwd, name);
if (exists(file)) return { passed: false, score: 0 };
fs.writeFileSync(file, source, { flag: 'wx' });
const result = spawnSync(process.execPath, checkArguments(fs.realpathSync(cwd), name, step !== null), { cwd, encoding: 'utf8',
env: { LANG: 'C.UTF-8' }, shell: false, timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 65536 });
// Grader files never linger: in stepped tasks the workspace accumulates, and a later ticket's
// agent could read or replay an earlier grader. The planted-grader guard above still applies.
fs.rmSync(file, { force: true });
const passed = result.status === 0 && !result.error;
let score = passed ? 1 : 0;
const match = SCORE_LINE.exec(result.stdout || '');
// A grader that advertises ECC_EVAL_SCORE but never printed it died mid-run (e.g. the graded
// server crashed the process): that is a zero, never a silent pass. A printed but malformed
// line keeps the exit-status score.
const graderDied = passed && !match && source.includes('ECC_EVAL_SCORE')
&& !(result.stdout || '').includes('ECC_EVAL_SCORE');
if (passed && match) {
try {
const parsed = JSON.parse(match[1]);
if (typeof parsed?.score === 'number' && parsed.score >= 0 && parsed.score <= 1) score = parsed.score;
} catch { /* A malformed score line keeps the exit-status score. */ }
}
if (graderDied) score = 0;
return { passed, score };
}
function runCheck(cwd, source) { return runScoredCheck(cwd, source).passed; }
function writeWorkspace(cwd, files) {
for (const [relative, content] of Object.entries(files)) {
fs.mkdirSync(path.dirname(path.join(cwd, relative)), { recursive: true });
fs.writeFileSync(path.join(cwd, relative), content, { flag: 'wx' });
}
}
function wilson(successes, n) {
if (!n) return [0, 1];
const z = 1.959963984540054;
const p = successes / n;
const denominator = 1 + z * z / n;
const center = (p + z * z / (2 * n)) / denominator;
const radius = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denominator;
return [Math.max(0, center - radius), Math.min(1, center + radius)];
}
function summarize(outcomes, arms = ARMS) {
const ids = [...new Set(outcomes.map(row => row.id))];
// Reference arm: full when present (all-arms runs), otherwise the last registered arm (baseline in subset runs).
const reference = arms.includes('full') ? 'full' : arms[arms.length - 1];
const rates = arms.map(arm => {
const rows = outcomes.filter(row => row.arm === arm);
return { arm, attempts: rows.length, successes: rows.filter(row => row.passed).length,
rate: rows.length ? rows.filter(row => row.passed).length / rows.length : null,
meanScore: rows.length ? rows.reduce((sum, row) => sum + (typeof row.score === 'number' ? row.score : Number(row.passed)), 0) / rows.length : null };
});
const pairs = arms.filter(arm => arm !== reference).map(arm => {
const differences = ids.map(id => {
const rows = outcomes.filter(row => row.id === id);
const baseline = rows.filter(row => row.arm === reference);
const delta = baseline.map(row => Number(rows.find(r => r.arm === arm && r.repeat === row.repeat)?.passed === true)
- Number(row.passed === true));
return delta.length ? delta.reduce((a, b) => a + b, 0) / delta.length : null;
}).filter(value => value !== null);
const n = differences.length;
const delta = n ? differences.reduce((a, b) => a + b, 0) / n : null;
// Paired task-cluster means in [-1,1]. Hoeffding with Bonferroni for the arm comparisons.
const radius = n ? Math.sqrt(2 * Math.log(80) / n) : 2;
return { arm, reference, n, delta, interval: [Math.max(-1, (delta || 0) - radius), Math.min(1, (delta || 0) + radius)],
method: 'paired-task-cluster-hoeffding-familywise-95' };
});
return { distinctTasks: ids.length, rates, pairs };
}
function selectionTask(item) {
return { sessionId: 'ecc-eval', taskId: item.id, revision: 1, phase: 'evaluate', query: item.query,
...(item.noWorkflow === undefined ? {} : { noWorkflow: item.noWorkflow }),
...(item.explicitIds ? { explicitIds: item.explicitIds } : {}) };
}
function failureCode(error) {
if (['call-budget', 'deadline', 'source-drift', 'environment-drift', 'provider-failed', 'invalid-jsonl'].includes(error?.code)) return error.code;
for (const [code, pattern] of Object.entries(BLOCKS)) if (pattern.test(error?.message || '')) return code;
return 'evaluation-failed';
}
function fail(code) { const error = new Error(code); error.code = code; throw error; }
function launchEnvironment(launch) {
return { PATH: process.env.PATH, HOME: launch.home,
...(launch.codexHome ? { CODEX_HOME: launch.codexHome } : {}),
...(launch.claudeConfigDir ? { CLAUDE_CONFIG_DIR: launch.claudeConfigDir } : {}),
TMPDIR: launch.home, LANG: 'C.UTF-8' };
}
// One provider call. A throwing provider still records its elapsed time and
// restores the environment; only source drift propagates as itself.
function callProvider(state, environment, request, started) {
try {
return state.provider(request);
} catch (error) {
state.metrics.push({ phase: request.phase, elapsedMs: Date.now() - started, usage: null });
if (error?.code === 'source-drift') throw error;
return fail('provider-failed');
} finally { environment.restore(); }
}
function parseProviderOutput(state, raw) {
const parsed = state.family === 'claude' ? parseClaudeJson(raw?.stdout) : parseCodexJsonl(raw?.stdout);
const succeeded = raw?.status === 0 && !raw?.error;
return { parsed, succeeded };
}
function adapterCall(state, cwd, environment, args, options) {
if (state.calls >= state.maxCalls) fail('call-budget');
state.assertCurrent();
environment.verify();
const remaining = state.deadline - Date.now();
if (remaining <= 0) fail('deadline');
const phase = options.phase || (args.includes('read-only') ? 'selection' : 'task');
state.calls++;
const started = Date.now();
// Coding tasks outgrow the launcher's interactive default, so the evaluator's own call bound governs them.
const timeoutMs = Math.min(phase === 'task' ? state.callTimeoutMs : options.timeout, state.callTimeoutMs, remaining);
const env = options.env || launchEnvironment(environment.launch);
const raw = callProvider(state, environment,
{ phase, input: options.input, cwd, env, timeoutMs, maxBuffer: 1024 * 1024 }, started);
const elapsedMs = Date.now() - started;
const { parsed, succeeded } = parseProviderOutput(state, raw);
state.metrics.push({ phase, elapsedMs, usage: parsed.valid && succeeded ? parsed.usage : null });
if (Date.now() >= state.deadline || elapsedMs > timeoutMs) fail('deadline');
state.assertCurrent();
if (!succeeded) fail('provider-failed');
if (!parsed.valid) fail(parsed.error ? 'provider-failed' : 'invalid-jsonl');
return { status: 0, stdout: parsed.text };
}
function executeAdapter(state, cwd, environment) {
return (_command, args, options) => adapterCall(state, cwd, environment, args, options);
}
function selectionProbe(item, repoRoot, execute, environment, target) {
const options = { repoRoot, task: selectionTask(item), exclude: item.exclude || [], load: true };
try {
let selection = resolveTaskContext(options);
if (selection.reason === 'agent-selection-required') {
const proposedIds = proposeTaskContext({ target, query: item.query, candidates: selection.candidates, execute,
executable: environment.launch.codexPath || environment.launch.claudePath });
// An empty proposal is an explicit decline: honor it (inject nothing).
// The tier-2 fallback only applies when a non-empty proposal admitted
// nothing — never to override a decline.
const declined = proposedIds.length === 0;
const next = resolveTaskContext({ ...options, task: { ...options.task, proposedIds, noWorkflow: declined } });
if (next.selectedIds.length) selection = next;
else if (declined) selection = { ...next, reason: 'agent-declined-selection' };
else selection = resolveDeclinedFallback(options, selection);
}
return { id: item.id, category: item.category, passed: !item.expectedBlock
&& isDeepStrictEqual(selection.selectedIds, item.expectedIds), selectedIds: selection.selectedIds, failure: null };
} catch (error) {
const failure = failureCode(error);
return { id: item.id, category: item.category, passed: Boolean(item.expectedBlock && failure === item.expectedBlock),
selectedIds: [], failure };
}
}
// Full relies on native discovery of the whole install; the Lean arms receive ECC-selected skill bodies;
// ecc-legacy runs bare against the pinned pre-scoping skill library; Baseline runs the bare task query.
// Stepped tasks run each ticket in the same accumulating workspace, grading after every step.
// `trial` carries repoRoot, execute, cwd, environment, target, harvest (or null)
// and metrics (or null) for one task trial.
function launchTrialStep(item, arm, trial, query, manualIds) {
const task = { sessionId: 'ecc-eval', taskId: item.id, revision: 1, phase: 'evaluate', query };
return launchTaskContext({ repoRoot: trial.repoRoot, execute: trial.execute,
nativeEnvironment: trial.environment.launch, target: trial.target,
bare: arm === 'baseline' || arm === 'ecc-legacy',
task: { ...task, ...(arm === 'manual-lean' && manualIds?.length ? { explicitIds: manualIds } : {}) },
profileId: arm === 'full' ? 'full@1' : 'lean@1', selectionMode: arm === 'auto-lean' ? 'auto' : 'manual' });
}
function stepMetrics(metrics, start) {
return metrics ? metricsSince(metrics, start) : {};
}
function singleStepTrial(item, arm, repeat, trial) {
const { cwd, environment, harvest } = trial;
const result = launchTrialStep(item, arm, trial, item.query, item.manualIds);
if (harvest) harvest(arm, item.id, repeat, environment);
const verdict = runScoredCheck(cwd, item.check, item.checkTimeoutMs);
const passed = result.status === 'completed' && verdict.passed && verdict.score >= 0.999;
return { id: item.id, arm, repeat, passed, score: result.status === 'completed' ? verdict.score : 0,
selectedIds: result.selection.selectedIds, failure: passed ? null : 'hidden-check' };
}
function steppedTrial(item, arm, repeat, trial) {
const { cwd, environment, harvest, metrics } = trial;
const steps = [];
const selectedIds = [];
for (let index = 0; index < item.steps.length; index++) {
const step = item.steps[index];
const start = metrics ? metrics.length : 0;
const result = launchTrialStep(item, arm, trial, step.query, step.manualIds || item.manualIds);
if (harvest) harvest(arm, `${item.id}--step${index + 1}`, repeat, environment);
if (result.status !== 'completed') {
// A failed ticket ends the chain; remaining tickets are unscored.
steps.push({ score: 0, ...stepMetrics(metrics, start) });
for (let rest = index + 1; rest < item.steps.length; rest++) {
steps.push({ score: 0, ...stepMetrics(metrics, metrics ? metrics.length : 0) });
}
break;
}
selectedIds.push(...result.selection.selectedIds);
const verdict = runScoredCheck(cwd, step.check, step.checkTimeoutMs, index + 1);
steps.push({ score: verdict.passed ? verdict.score : 0, ...stepMetrics(metrics, start) });
}
const score = steps.reduce((sum, step) => sum + step.score, 0) / item.steps.length;
const passed = steps.length === item.steps.length && steps.every(step => step.score >= 0.999);
return { id: item.id, arm, repeat, passed, score, selectedIds: [...new Set(selectedIds)], steps,
failure: passed ? null : 'hidden-check' };
}
function outcomeTrial(item, arm, repeat, trial) {
try {
return item.steps ? steppedTrial(item, arm, repeat, trial) : singleStepTrial(item, arm, repeat, trial);
} catch (error) {
if (trial.harvest) trial.harvest(arm, item.id, repeat, trial.environment);
return { id: item.id, arm, repeat, passed: false, score: 0, selectedIds: [], failure: failureCode(error) };
}
}
function metricsSince(metrics, start) {
const calls = metrics.slice(start);
const complete = calls.length > 0 && calls.every(call => call.usage !== null);
return { calls: calls.length, elapsedMs: calls.reduce((sum, c) => sum + c.elapsedMs, 0),
usage: complete ? calls.reduce((sum, c) => ({ inputTokens: sum.inputTokens + c.usage.inputTokens,
cachedInputTokens: sum.cachedInputTokens + c.usage.cachedInputTokens,
outputTokens: sum.outputTokens + c.usage.outputTokens }), { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 }) : null };
}
// Transcript retention is opt-in (--artifact-dir) and file-only: reports never embed session content or paths.
function createHarvester(artifactDir, envs) {
if (typeof artifactDir !== 'string' || !path.isAbsolute(artifactDir)) throw new Error('Artifact directory must be absolute');
fs.mkdirSync(artifactDir, { recursive: true });
const sessionsOf = env => {
const config = env.launch.claudeConfigDir;
const projects = config ? path.join(config, 'projects') : null;
if (!projects || !exists(projects)) return new Set();
const found = new Set();
const walk = directory => {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
const item = path.join(directory, entry.name);
if (entry.isDirectory()) walk(item);
else if (entry.name.endsWith('.jsonl')) found.add(item);
}
};
walk(projects);
return found;
};
const seen = new Map(Object.entries(envs).map(([name, env]) => [name, sessionsOf(env)]));
const index = [];
return {
record(arm, id, repeat, env) {
const before = seen.get(arm) || new Set();
const now = sessionsOf(env);
seen.set(arm, now);
const fresh = [...now].filter(file => !before.has(file));
if (!fresh.length) return;
const directory = path.join(artifactDir, `${id}--${arm}--${repeat}`);
fs.mkdirSync(directory, { recursive: true });
for (const file of fresh) fs.copyFileSync(file, path.join(directory, path.basename(file)));
index.push({ id, arm, repeat, files: fresh.map(file => path.basename(file)) });
},
writeIndex() { fs.writeFileSync(path.join(artifactDir, 'artifact-index.json'), `${JSON.stringify(index, null, 1)}\n`); },
};
}
// Validate the caller's bounds and opt-ins and settle on the provider family.
function resolveEvaluationFamily(options) {
const { provider, allowRealProvider, maxCalls, deadlineMs, callTimeoutMs, registration,
family, executable, effort, allowCredentialedTools } = options;
if (!provider && !allowRealProvider) throw new Error('Evaluation requires an injected provider or explicit opt-in');
if (!bounded(maxCalls, 1, 2000) || !bounded(deadlineMs, 1, 8 * 3600000)
|| !bounded(callTimeoutMs, 1, 600000)) throw new Error('Invalid call or deadline bound');
if (!provider && !registration) throw new Error('Real evaluation requires prior registration');
const resolvedFamily = provider ? (family || 'codex') : resolveFamily(family, executable);
if (resolvedFamily === 'claude' && effort !== undefined) throw new Error('Reasoning effort applies only to the Codex provider');
if (!provider && resolvedFamily === 'claude' && !allowCredentialedTools) {
throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required');
}
return resolvedFamily;
}
function pinEvaluation(options, resolvedFamily) {
const { repoRoot, corpus, repeats, model, executable, effort, arms, provider, registration } = options;
const pin = preregister({ repoRoot, corpus, repeats, model, executable, effort, arms });
if (!provider && resolvedFamily === 'codex' && pin.arms.includes('ecc-legacy')) {
throw new Error('Codex real evaluation requires --arms without ecc-legacy; the pinned legacy skills arm is Claude-only');
}
if (registration && !isDeepStrictEqual(registration, pin)) throw new Error('Registration pin mismatch');
return pin;
}
function createLiveProvider(resolvedFamily, options) {
const { allowRealProvider, allowCredentialedTools, executable, model, effort, authHome, artifactDir } = options;
return resolvedFamily === 'claude'
? createClaudeProvider({ allowRealProvider, allowCredentialedTools, executable, model,
persistSessions: Boolean(artifactDir) })
: createCodexProvider({ allowRealProvider, executable, model, effort, authHome });
}
function createEvaluationState(options, plan) {
const { repoRoot, corpus, maxCalls, callTimeoutMs, deadlineMs } = options;
const { pin, resolvedFamily, liveProvider } = plan;
return { calls: 0, metrics: [], maxCalls, callTimeoutMs, family: resolvedFamily,
deadline: Date.now() + deadlineMs, provider: liveProvider,
assertCurrent() {
if (digestObject(corpus) !== pin.corpusDigest || sourceSnapshot(repoRoot).sourceDigest !== pin.sourceDigest) fail('source-drift');
} };
}
function planEvaluation(options) {
const resolvedFamily = resolveEvaluationFamily(options);
const pin = pinEvaluation(options, resolvedFamily);
const injected = Boolean(options.provider);
const liveProvider = options.provider || createLiveProvider(resolvedFamily, options);
const plan = { resolvedFamily, pin, injected, liveProvider };
plan.state = createEvaluationState(options, plan);
return plan;
}
function prepareEvaluationEnvironments(options, plan, installRoot) {
const { repoRoot, executable } = options;
const { injected, resolvedFamily, pin } = plan;
if (injected) return syntheticEnvironments(installRoot);
if (resolvedFamily !== 'claude') return prepareEnvironments({ repoRoot, executable, root: installRoot });
return prepareClaudeEnvironments({ repoRoot, executable, root: installRoot,
...(pin.arms.includes('ecc-legacy')
? { legacySource: exportLegacySource({ repoRoot, destination: path.join(installRoot, 'legacy-source') }) }
: {}) });
}
function describeInstalls(envs) {
return Object.fromEntries(Object.entries(envs).map(([name, env]) => [name,
{ profileId: env.profileId, skills: env.skills, ...(env.sourceSha ? { sourceSha: env.sourceSha } : {}) }]));
}
function runSelectionProbes(run, selection) {
const { corpus, repoRoot, state, temp, envs, resolvedFamily } = run;
for (const item of corpus.selection) {
const cwd = path.join(temp, `${item.id}--selection`);
fs.mkdirSync(cwd);
const start = state.metrics.length;
selection.push({ ...selectionProbe(item, repoRoot, executeAdapter(state, cwd, envs.lean), envs.lean, resolvedFamily),
...metricsSince(state.metrics, start) });
}
}
function runOutcomeTrials(run, outcomes) {
const { corpus, pin, repoRoot, state, temp, envs, resolvedFamily, harvest } = run;
for (const scheduled of pin.order) {
const item = corpus.tasks.find(c => c.id === scheduled.id);
for (const arm of scheduled.arms) {
const cwd = path.join(temp, `${item.id}--${arm}--${scheduled.repeat}`);
const environment = envs[['full', 'baseline', 'ecc-legacy'].includes(arm) ? arm : 'lean'];
fs.mkdirSync(cwd);
writeWorkspace(cwd, item.files);
const start = state.metrics.length;
const trial = { repoRoot, execute: executeAdapter(state, cwd, environment), cwd, environment,
target: resolvedFamily, harvest, metrics: state.metrics };
outcomes.push({ ...outcomeTrial(item, arm, scheduled.repeat, trial), ...metricsSince(state.metrics, start) });
fs.rmSync(cwd, { recursive: true, force: true });
}
}
}
// Prepare installs in a private temp root, run every selection probe and task
// trial, and always remove the root afterwards.
function executeEvaluation(options, plan) {
const { repoRoot, corpus, environments, artifactDir } = options;
const { resolvedFamily, pin, injected, state } = plan;
const temp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-ai-eval-')));
const selection = [];
const outcomes = [];
let installs = null;
let harvester = null;
try {
const installRoot = path.join(temp, 'installs');
fs.mkdirSync(installRoot, { mode: 0o700 });
const envs = environments || prepareEvaluationEnvironments(options, plan, installRoot);
installs = describeInstalls(envs);
harvester = artifactDir && resolvedFamily === 'claude' && !injected ? createHarvester(artifactDir, envs) : null;
const harvest = harvester ? (arm, id, repeat, env) => harvester.record(arm, id, repeat, env) : null;
const run = { corpus, pin, repoRoot, state, temp, envs, resolvedFamily, harvest };
runSelectionProbes(run, selection);
runOutcomeTrials(run, outcomes);
if (harvester) harvester.writeIndex();
} finally { if (harvester) harvester.writeIndex(); fs.rmSync(temp, { recursive: true, force: true }); }
return { selection, outcomes, installs, harvester };
}
function summarizeSelection(selection) {
const selectionSuccesses = selection.filter(row => row.passed).length;
return { n: selection.length, successes: selectionSuccesses,
categories: [...new Set(selection.map(row => row.category))].map(category => ({ category,
n: selection.filter(row => row.category === category).length,
successes: selection.filter(row => row.category === category && row.passed).length })),
interval: wilson(selectionSuccesses, selection.length), method: 'wilson-95-descriptive-purposive-sample' };
}
function buildEvaluationReport(options, plan, run) {
const { maxCalls, deadlineMs, callTimeoutMs } = options;
const { pin, injected, liveProvider, state, resolvedFamily } = plan;
const { selection, outcomes, installs, harvester } = run;
const summary = summarize(outcomes, pin.arms);
const insufficient = summary.distinctTasks < pin.minimumDistinctTasks || selection.length < pin.minimumDistinctTasks;
return { schemaVersion: 'ecc.context-eval.v2', registration: pin,
evidence: injected ? 'injected-provider' : resolvedFamily === 'claude' ? 'claude-json' : 'codex-jsonl', installs,
authentication: injected ? 'injected' : liveProvider.authentication, credentialsRetained: false,
calls: state.calls, bounds: { maxCalls, deadlineMs, callTimeoutMs }, selection, outcomes, summary,
selectionSummary: summarizeSelection(selection),
gate: { status: insufficient ? 'insufficient-sample' : injected ? 'synthetic-only' : 'review-required',
nonInferioritySupported: !insufficient && !injected && summary.pairs.every(p => p.interval[0] >= -pin.nonInferiorityMargin),
releaseApproved: false }, nativeInvocation: 'unobserved',
measurementScope: 'native-install-hidden-graded-coding-tasks',
artifactRetention: harvester ? 'session-jsonl-per-task-trial' : 'none', ...metricsSince(state.metrics, 0) };
}
function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), registration,
repeats = 1, provider, family, allowRealProvider = false, allowCredentialedTools = false,
executable, model, effort, authHome, environments,
arms = undefined, artifactDir = null, maxCalls = 300, deadlineMs = 3600000, callTimeoutMs = 300000 } = {}) {
const options = { repoRoot, corpus, registration, repeats, provider, family, allowRealProvider,
allowCredentialedTools, executable, model, effort, authHome, environments, arms, artifactDir,
maxCalls, deadlineMs, callTimeoutMs };
const plan = planEvaluation(options);
const run = executeEvaluation(options, plan);
return buildEvaluationReport(options, plan, run);
}
module.exports = { loadCorpus, preregister, runEvaluation, parseCodexJsonl, parseClaudeJson, summarize, wilson,
runCheck, runScoredCheck, createAuthLease, createCodexProvider, createClaudeProvider, prepareEnvironments,
prepareClaudeEnvironments, exportLegacySource, providerFamily, resolveFamily, readClaudeKeychainToken };