scripts/evaluate-slack-approval-intent.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
// Slack approval intent calibration. Offline (default): the corpus contract
// plus a deterministic smoke of the production side call against a stubbed
// provider endpoint, which also proves no change content reaches the judge.
// --live: the same side call against real models, with a private report and
// a gate on false approvals and recall.
import { randomUUID } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { parseArgs, parseEnv } from 'node:util';
import { configureModelAccessResolver } from '../src/config/model-access.ts';
import { approvalChangeSummary } from '../src/management/slack-approval.ts';
import { SqliteSettingsStore } from '../src/config/settings-store.ts';
import { bootstrapRuntimeProviders } from '../src/runtime-bootstrap.ts';
import {
classifySlackManagementApproval,
shouldResolveSlackManagementApproval,
SLACK_MANAGEMENT_APPROVAL_INSTRUCTIONS,
slackManagementApprovalMessage,
slackSteeringCommand,
} from '../src/slack/interaction-intent.ts';
import { assertNodeVersion } from './lib/node-version.mjs';
import { outsideGit } from './lib/private-evidence.mjs';
const CORPUS_PATH = new URL('../evals/slack-approval-intent/cases.json', import.meta.url);
const LABELS = ['approve', 'decline', 'other'];
const MINIMUM_CASES = { approve: 15, decline: 8, other: 12 };
const REQUIRED_OTHER_CATEGORIES = ['conditional', 'question', 'quote', 'injection'];
const SMOKE_MODEL = 'anthropic/claude-sonnet-5';
const MODEL_ENV = 'SLACK_APPROVAL_EVAL_MODEL';
// A false decline retires a change the person can ask for again; a false approval applies one.
const TARGETS = { falseApprovalsMax: 0, falseDeclinesMax: 1, approveRecallMin: 0.9, declineRecallMin: 0.75, failuresMax: 0 };
// Only provider credentials are ever read from --env-file; nothing is printed.
const PROVIDER_ENV = { anthropic: 'ANTHROPIC_API_KEY', openai: 'OPENAI_API_KEY', openrouter: 'OPENROUTER_API_KEY' };
function assert(condition, message) {
if (!condition) throw new Error(message);
}
// ── Corpus ─────────────────────────────────────────────────────────────────
function validateCorpus(corpus) {
assert(corpus.schemaVersion === 1, 'Unknown corpus schema version.');
const ids = new Set();
const counts = { approve: 0, decline: 0, other: 0 };
const otherCategories = new Set();
for (const entry of corpus.cases) {
assert(typeof entry.id === 'string' && !ids.has(entry.id), `Duplicate or missing case id: ${entry.id}`);
ids.add(entry.id);
assert(LABELS.includes(entry.expected), `${entry.id}: expected must be one of ${LABELS.join(', ')}.`);
assert(typeof entry.text === 'string' && entry.text.trim(), `${entry.id}: text is required.`);
assert(Array.isArray(corpus.changes[entry.change]), `${entry.id}: unknown change ${entry.change}.`);
assert(!shouldResolveSlackManagementApproval(entry.text) && !slackSteeringCommand(entry.text),
`${entry.id}: the host decides an exact approval word or a stop phrase without the judge.`);
counts[entry.expected] += 1;
if (entry.expected === 'other') otherCategories.add(entry.category);
}
for (const label of LABELS) {
assert(counts[label] >= MINIMUM_CASES[label], `The corpus needs at least ${MINIMUM_CASES[label]} ${label} cases.`);
}
for (const category of REQUIRED_OTHER_CATEGORIES) {
assert(otherCategories.has(category), `The corpus needs an other case in category ${category}.`);
}
return counts;
}
/** What the host gives the judge for a case: the message and its summary of the change, never the change. */
function caseInput(corpus, entry) {
return { text: entry.text, change: approvalChangeSummary({ operations: corpus.changes[entry.change] }) };
}
/** Every string a change carries, which must never appear in the judge's request. */
function changeContent(operations) {
const strings = [];
const walk = (value) => {
if (typeof value === 'string') strings.push(value);
else if (Array.isArray(value)) value.forEach(walk);
else if (value && typeof value === 'object') Object.values(value).forEach(walk);
};
operations.forEach((operation) => walk(operation.patch ?? {}));
return strings.filter((value) => value.length >= 8);
}
// ── Offline smoke ──────────────────────────────────────────────────────────
function anthropicStream(text) {
const usage = { input_tokens: 10, output_tokens: 1 };
return [
{ type: 'message_start', message: { id: 'msg_1', type: 'message', role: 'assistant', model: 'm', content: [], usage } },
{ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } },
{ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } },
{ type: 'content_block_stop', index: 0 },
{ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 8 } },
{ type: 'message_stop' },
].map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join('');
}
/** Every request goes to this stub; it answers `answer(body)`, or fails with HTTP 500 when that is undefined. */
function stubProvider(answer) {
const bodies = [];
const original = globalThis.fetch;
globalThis.fetch = async (input, init) => {
const body = await new Request(input, init).json();
bodies.push(body);
const text = answer(body);
return text === undefined
? new Response('{}', { status: 500 })
: new Response(anthropicStream(text), { status: 200, headers: { 'content-type': 'text/event-stream' } });
};
return { bodies, restore: () => { globalThis.fetch = original; } };
}
function requestTexts(body) {
const blockText = (content) => typeof content === 'string'
? content
: content.filter((block) => block.type === 'text').map((block) => block.text).join('');
return {
system: Array.isArray(body.system) ? blockText(body.system) : body.system,
messages: body.messages.map(({ role, content }) => ({ role, text: blockText(content) })),
};
}
async function runOfflineSmoke(corpus, settings) {
// A stand-in key: every request below goes to the stub, never to Anthropic.
process.env.ANTHROPIC_API_KEY = 'sk-test-not-a-key';
configureModelAccessResolver({ resolve: async () => ({ apiKey: 'sk-test-not-a-key' }) });
const judge = (input) => classifySlackManagementApproval(
{ ...input, requestedModel: SMOKE_MODEL }, { timeoutMs: 10_000, settings });
const smoke = [];
for (const entry of corpus.cases) {
const input = caseInput(corpus, entry);
const provider = stubProvider(() => JSON.stringify({ approval: entry.expected }));
let judgment;
try {
judgment = await judge(input);
} finally {
provider.restore();
}
assert(provider.bodies.length === 1, `${entry.id}: expected one side call, saw ${provider.bodies.length}.`);
const sent = requestTexts(provider.bodies[0]);
assert(sent.system === SLACK_MANAGEMENT_APPROVAL_INSTRUCTIONS, `${entry.id}: the instructions were not sent as the system prompt.`);
assert(sent.messages.length === 1 && sent.messages[0].role === 'user' &&
sent.messages[0].text === slackManagementApprovalMessage(input),
`${entry.id}: the judge must see exactly the person's message and the host's summary.`);
const read = [sent.system, ...sent.messages.map(({ text }) => text)].join('\n');
for (const content of changeContent(corpus.changes[entry.change])) {
assert(!read.includes(content) && !read.includes(JSON.stringify(content).slice(1, -1)),
`${entry.id}: change content reached the judge: ${content}`);
}
assert(judgment.approval === entry.expected && judgment.failed === false,
`${entry.id}: the side call answer was not parsed (${JSON.stringify(judgment)}).`);
smoke.push({ caseId: entry.id, judged: judgment.approval });
}
for (const [name, answer] of [
['provider error', () => undefined],
['extra field', () => '{"approval":"approve","because":"they said yes"}'],
['prose', () => 'The person approves.'],
]) {
const provider = stubProvider(answer);
try {
const judgment = await judge({ text: 'sure', change: caseInput(corpus, corpus.cases[0]).change });
assert(judgment.approval === 'other', `A ${name} must judge other, not ${judgment.approval}.`);
} finally {
provider.restore();
}
}
return smoke;
}
// ── Live ───────────────────────────────────────────────────────────────────
function loadProviderEnv(files) {
const loadable = new Set(Object.values(PROVIDER_ENV));
const loaded = [];
for (const file of files) {
for (const [name, value] of Object.entries(parseEnv(readFileSync(resolve(file), 'utf8')))) {
if (!loadable.has(name) || !value || process.env[name]) continue;
process.env[name] = value;
loaded.push(name);
}
}
return [...new Set(loaded)].sort();
}
function credentialName(model) {
return PROVIDER_ENV[model.slice(0, model.indexOf('/'))];
}
function summarize(results) {
const matrix = Object.fromEntries(LABELS.map((label) => [label, { approve: 0, decline: 0, other: 0 }]));
let failures = 0;
for (const result of results) {
matrix[result.expected][result.judged] += 1;
if (result.failed) failures += 1;
}
const total = (label) => LABELS.reduce((sum, judged) => sum + matrix[label][judged], 0);
const falseApprovals = matrix.decline.approve + matrix.other.approve;
const falseDeclines = matrix.approve.decline + matrix.other.decline;
const approveRecall = total('approve') ? matrix.approve.approve / total('approve') : 0;
const declineRecall = total('decline') ? matrix.decline.decline / total('decline') : 0;
const gateFailures = [
...(falseApprovals > TARGETS.falseApprovalsMax ? ['false approvals'] : []),
...(falseDeclines > TARGETS.falseDeclinesMax ? ['false declines'] : []),
...(approveRecall < TARGETS.approveRecallMin ? ['approve recall'] : []),
...(declineRecall < TARGETS.declineRecallMin ? ['decline recall'] : []),
...(failures > TARGETS.failuresMax ? ['side-call failures'] : []),
];
return { matrix, falseApprovals, falseDeclines, approveRecall, declineRecall, failures, gate: { passed: gateFailures.length === 0, failures: gateFailures } };
}
async function runLive(corpus, options, settings) {
const createdAt = new Date().toISOString();
// Refuse a report path inside a checkout before any paid call.
const path = outputPath(options.output, createdAt);
const loaded = loadProviderEnv(options.envFiles);
configureModelAccessResolver({
resolve: async (grant) => {
const key = process.env[PROVIDER_ENV[grant.providerId]];
if (!key) throw new Error(`No credential for ${grant.providerId}.`);
return { apiKey: key };
},
});
const models = options.models.length ? options.models : (process.env[MODEL_ENV] ?? '').split(',').filter(Boolean);
assert(models.length > 0, `Name a model with --model or ${MODEL_ENV}.`);
const cases = options.caseIds.length
? corpus.cases.filter(({ id }) => options.caseIds.includes(id))
: corpus.cases;
assert(cases.length > 0, 'No case matches --case.');
const report = { kind: 'slack-approval-intent live calibration', corpusVersion: corpus.corpusVersion, createdAt, source: sourceFingerprint(), loadedEnv: loaded, targets: TARGETS, models: [] };
for (const model of models) {
const missing = credentialName(model);
if (!missing || !process.env[missing]) {
report.models.push({ model, skipped: missing ? `${missing} is not set` : 'unsupported provider', gate: { passed: false, failures: ['skipped'] } });
continue;
}
const results = [];
for (const entry of cases) {
const startedAt = performance.now();
const judgment = await classifySlackManagementApproval(
{ ...caseInput(corpus, entry), requestedModel: model }, { timeoutMs: options.timeoutMs, settings });
results.push({
caseId: entry.id, category: entry.category, expected: entry.expected, judged: judgment.approval,
failed: judgment.failed, durationMs: Math.round(performance.now() - startedAt),
...(judgment.result ? { raw: judgment.result.text, usage: judgment.result.reportedUsage } : {}),
});
process.stdout.write(`${model} ${entry.id}: ${judgment.approval}${judgment.approval === entry.expected ? '' : ` (expected ${entry.expected})`}\n`);
}
report.models.push({ model, ...summarize(results), results });
}
report.gate = { passed: report.models.every(({ gate }) => gate.passed) };
writePrivateJson(path, report);
for (const entry of report.models) {
process.stdout.write(entry.skipped
? `${entry.model}: skipped (${entry.skipped})\n`
: `${entry.model}: false approvals ${entry.falseApprovals}, false declines ${entry.falseDeclines}, approve recall ${entry.approveRecall.toFixed(2)}, decline recall ${entry.declineRecall.toFixed(2)}, failures ${entry.failures}; gate ${entry.gate.passed ? 'passed' : `failed (${entry.gate.failures.join(', ')})`}\n`);
}
process.stdout.write(`Wrote the private report to ${path}\n`);
return report;
}
function outputPath(option, createdAt) {
const target = option
? resolve(option)
: join(homedir(), '.chickpea', 'evals', 'slack-approval-intent', createdAt.replace(/[:.]/g, '-'), 'report.json');
return outsideGit(target.endsWith('.json') ? target : join(target, 'report.json'));
}
function writePrivateJson(path, value) {
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
const temporary = `${path}.${randomUUID()}.pending`;
writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx', mode: 0o600 });
renameSync(temporary, path);
}
function sourceFingerprint() {
try {
const root = new URL('..', import.meta.url).pathname;
const head = execFileSync('git', ['-C', root, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
const dirty = execFileSync('git', ['-C', root, 'status', '--porcelain'], { encoding: 'utf8' }).trim().length > 0;
return { head, dirty };
} catch {
return { head: null, dirty: null };
}
}
// ── CLI ────────────────────────────────────────────────────────────────────
function parseOptions(argv) {
const { values } = parseArgs({
args: argv,
options: {
live: { type: 'boolean' },
help: { type: 'boolean' },
model: { type: 'string', multiple: true },
case: { type: 'string', multiple: true },
'timeout-ms': { type: 'string' },
output: { type: 'string' },
'env-file': { type: 'string', multiple: true },
},
});
const liveOnly = ['model', 'case', 'timeout-ms', 'output', 'env-file'];
if (!values.live) {
const used = liveOnly.filter((name) => values[name] !== undefined);
assert(used.length === 0, `--${used[0]} requires --live.`);
}
const timeoutMs = values['timeout-ms'] === undefined ? 30_000 : Number(values['timeout-ms']);
assert(Number.isInteger(timeoutMs) && timeoutMs >= 1_000 && timeoutMs <= 120_000, '--timeout-ms must be an integer from 1000 to 120000.');
return {
live: values.live === true,
help: values.help === true,
models: (values.model ?? []).flatMap((model) => model.split(',')).filter(Boolean),
caseIds: values.case ?? [],
timeoutMs,
output: values.output,
envFiles: values['env-file'] ?? [],
};
}
function printHelp() {
process.stdout.write([
'Usage: npm run evaluate:slack-approval-intent',
' npm run evaluate:slack-approval-intent -- --live [--model ID ...] [--case ID ...] [--timeout-ms MS]',
' [--output PATH] [--env-file PATH ...]',
'',
'Offline (default): validates evals/slack-approval-intent/cases.json, then sends every case through the',
'production approval side call to a stubbed provider endpoint. It checks the judge sees only the',
'person\'s message and the host\'s fixed-label summary of the change, that no change content reaches the',
'request, that answers parse, and that a provider error and a malformed answer behave. No case may be an',
'exact approval word or a stop phrase, because the host decides those without the judge.',
'No network, no credentials, and no real model: it does not measure judgment.',
'',
`Live: judges every selected case with each model from --model (repeatable or comma-separated) or`,
`${MODEL_ENV}, through the same side call the host runs. Providers: anthropic, openai, openrouter;`,
'credentials come from ANTHROPIC_API_KEY, OPENAI_API_KEY or OPENROUTER_API_KEY, and --env-file loads only',
'those names. The report holds the cases\' raw answers, so it is private: it defaults to',
'~/.chickpea/evals/slack-approval-intent/<time>/report.json and is refused inside any Git checkout.',
`Exits 1 unless false approvals = ${TARGETS.falseApprovalsMax}, false declines <= ${TARGETS.falseDeclinesMax},`,
`approve recall >= ${TARGETS.approveRecallMin},`,
`decline recall >= ${TARGETS.declineRecallMin} and side-call failures = ${TARGETS.failuresMax} for every model.`,
'',
].join('\n'));
}
async function main() {
process.env.DO_NOT_TRACK ??= '1';
const options = parseOptions(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
assertNodeVersion();
const corpus = JSON.parse(readFileSync(CORPUS_PATH, 'utf8'));
const counts = validateCorpus(corpus);
bootstrapRuntimeProviders();
const settings = new SqliteSettingsStore(':memory:');
try {
if (options.live) {
const report = await runLive(corpus, options, settings);
if (!report.gate.passed) process.exitCode = 1;
return;
}
const smoke = await runOfflineSmoke(corpus, settings);
process.stdout.write(`${JSON.stringify({
kind: 'slack-approval-intent offline check: corpus contract and the production side call against a stubbed provider; no real model',
corpusVersion: corpus.corpusVersion,
corpus: counts,
smoke: smoke.length,
passed: true,
}, null, 2)}\n`);
} finally {
settings.close();
}
}
main().catch((error) => {
process.stderr.write(`Slack approval intent evaluation failed: ${error instanceof Error ? error.message : String(error)}\n`);
process.exitCode = 1;
});