scripts/eval-selection.ts
Rama main · réplica de lectura del repositorio original.
// Checks the selection against your own labelled samples (a "gold set"): each case runs the site's
// selection steps (editorial/analyze.ts) — the prefilter, then the score prompt twice with every model in
// --models, the two scores deciding against the case's tier threshold (gold cases carry no source class) — and is compared with your
// decision. A threshold sweep shows what another threshold would have done. The gold file has one case
// per line as eval-selection-core.ts describes; lines starting with // are skipped.
// Usage: node --env-file=.env scripts/eval-selection.ts [--gold .data/gold.jsonl] [--models default,deepseek-flash] [--n 200] [--split all] [--label "..."]
// Receipts make re-runs free; "either" cases are excluded from decisive metrics. Each run is also
// imported into SelectBench (admin → SelectBench) with every case, unless --no-import is given.
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
import path from "node:path";
import { parseArgs } from "node:util";
import { DEPLOYMENT } from "@aihot/site";
import { REPO_ROOT } from "@aihot/backend/config";
import { closeDb } from "@aihot/backend/db";
import {
SELECTION_PROMPT_VERSION,
awaitsMaterial,
buildScoreInput,
runSelectionPrefilter,
runSelectionScores,
scoresSelect,
selectionThreshold,
shownScore,
type AnalysisRun,
type AnalyzeInputArticle,
} from "@aihot/backend/editorial/analyze";
import { importSelectBenchRun } from "@aihot/backend/admin/selectbench";
import { evalModels, pmap, positiveInt, safeReportNamePart, sampleCases, usageFor } from "./eval-tools.ts";
import { parseSelectionGoldJsonl, selectionMetrics, selectionThresholdMetrics, type SelectionDecision, type SelectionGoldRow } from "./eval-selection-core.ts";
// Without options: the site's gold set (site.ts DEPLOYMENT.selectionGold), else the whole of .data/gold.jsonl
// (up to 200 cases), swept over a wide range of thresholds.
const own = DEPLOYMENT.selectionGold;
const defaults = own
? { gold: own.file, n: String(own.sample), split: own.split, sweep: own.sweep }
: { gold: ".data/gold.jsonl", n: "200", split: "all", sweep: [40, 90] };
const { values } = parseArgs({
options: {
gold: { type: "string", default: defaults.gold },
models: { type: "string" },
n: { type: "string", default: defaults.n },
split: { type: "string", default: defaults.split },
concurrency: { type: "string", default: "6" },
seed: { type: "string", default: "7" },
label: { type: "string" },
"no-import": { type: "boolean", default: false },
},
});
const rows = parseSelectionGoldJsonl(readFileSync(path.resolve(REPO_ROOT, values.gold!), "utf8"));
const sample = sampleCases(rows, { split: values.split!, n: positiveInt(values.n!, "n"), seed: Number(values.seed) });
const concurrency = positiveInt(values.concurrency!, "concurrency");
function toInput(r: SelectionGoldRow): AnalyzeInputArticle {
const m = r.material;
const isX = r.sourceFacts.sourceKind === "x_search";
const body = m.bodyOriginal || m.bodyZh || null;
return {
id: `gold-${r.caseId}`,
revision: 1,
bodyStatus: "ok",
title: m.originalTitle || m.title,
url: "https://example.invalid/" + r.caseId,
author: null,
publishedAt: m.publishedAt ? new Date(m.publishedAt) : null,
bodyText: isX ? null : body,
excerpt: null,
xPost: isX ? { authorName: m.sourceName, handle: "", text: body ?? m.title } : null,
media: [],
source: { name: m.sourceName, kind: r.sourceFacts.sourceKind, tier: r.sourceFacts.sourceTier ?? "T2", firstParty: r.sourceFacts.firstParty ?? false },
};
}
/**
* A case as the site's rules leave it: `relevance` is what the prefilter (or missing material) decided,
* `scores` the two scores held against the case's tier threshold. The eval writes no reader-facing copy;
* an item past the prefilter is taken as written up, as the site writes one for every such item.
*/
interface Outcome {
r: SelectionGoldRow;
relevance: "pass" | "block" | "unknown" | null;
scores: AnalysisRun["scores"];
receiptIds: number[];
error: string | null;
}
/** The site's decision for the case, at its tier threshold or at `threshold`; null when the case failed. */
const decide = (x: Outcome, threshold?: number): SelectionDecision | null =>
x.error !== null ? null : x.relevance === "pass" && scoresSelect(x.scores, threshold) ? "select" : "reject";
const models = await evalModels(values.models, "score");
const report: Record<string, unknown> = {};
for (const model of models) {
const started = performance.now();
// Two gold cases can have different source metadata / thresholds while rendering the same score prompt.
// Share only that score result (including a failure); prefilter and threshold semantics remain per case.
const scoreRequests = new Map<string, Promise<{ scores: AnalysisRun["scores"]; receiptIds: number[]; error: string | null }>>();
const results = await pmap(sample, concurrency, async (r): Promise<Outcome> => {
const input = toInput(r);
const receiptIds: number[] = [];
const outcome = (relevance: Outcome["relevance"], scores: AnalysisRun["scores"] = null, error: string | null = null): Outcome =>
({ r, relevance, scores, receiptIds, error });
// Only a title: the site waits for material, without a model call.
if (awaitsMaterial(input)) return outcome("unknown");
try {
const prefilter = await runSelectionPrefilter(input, {}, (id) => receiptIds.push(id));
if (prefilter.label === "BLOCK") return outcome("block");
const threshold = selectionThreshold(input.source);
if (threshold === null) return outcome("pass");
const key = buildScoreInput(input);
let request = scoreRequests.get(key);
if (!request) {
const sharedReceiptIds: number[] = [];
request = runSelectionScores(input, { scoreModel: model }, (id) => sharedReceiptIds.push(id)).then(
(scores) => ({ scores, receiptIds: sharedReceiptIds, error: null }),
(error: unknown) => ({ scores: null, receiptIds: sharedReceiptIds, error: String(error).slice(0, 200) }),
);
scoreRequests.set(key, request);
}
const shared = await request;
receiptIds.push(...shared.receiptIds);
if (shared.error) return outcome(null, null, shared.error);
// Model output is independent of source tier; the decision threshold is not.
return outcome("pass", shared.scores ? { ...shared.scores, threshold } : null);
} catch (error) {
return outcome(null, null, String(error).slice(0, 200));
}
});
const metrics = selectionMetrics(results.map((x) => ({ gold: x.r.gold.decision, decision: decide(x) })));
const mistakes: Array<Record<string, unknown>> = [];
for (const x of results) {
const pred = decide(x);
const gold = x.r.gold.decision;
if (pred === null || gold === "either") continue;
if (pred === "select" && gold === "reject") mistakes.push({ kind: "FP", title: x.r.material.title, score: shownScore(x.scores), stratum: x.r.samplingContext?.samplingStratum ?? null });
else if (pred === "reject" && gold === "select") mistakes.push({ kind: "FN", title: x.r.material.title, score: shownScore(x.scores), relevance: x.relevance, stratum: x.r.samplingContext?.samplingStratum ?? null });
}
const usage = await usageFor(results.flatMap((x) => x.receiptIds));
const summary = {
model,
...metrics,
...usage,
wallSeconds: Math.round((performance.now() - started) / 1000),
};
console.log(JSON.stringify(summary));
// Threshold sweep keeps valid-output metrics, while coverage and completeAcc charge decisive failures.
const thresholdCases = results.map((x) => ({ gold: x.r.gold.decision, available: x.error === null, passed: x.relevance === "pass", scores: x.scores }));
const sweep = [];
for (let t = defaults.sweep[0]; t <= defaults.sweep[1]; t += 2) {
sweep.push(selectionThresholdMetrics(thresholdCases, t));
}
console.log(sweep.map((s) =>
` t=${s.t} acc=${s.acc} completeAcc=${s.completeAcc} coverage=${s.coverage} P=${s.P} R=${s.R} F1=${s.F1} sel=${s.sel}`
).join("\n"));
const cases = results.map((x) => ({
caseId: x.r.caseId,
title: x.r.material.title,
stratum: x.r.samplingContext?.samplingStratum ?? null,
gold: x.r.gold.decision,
decision: decide(x),
score: shownScore(x.scores),
relevance: x.relevance,
receiptId: x.receiptIds[0] ?? null,
error: x.error,
}));
report[model] = { summary, sweep, mistakes, cases };
}
const outDir = path.join(REPO_ROOT, ".data/eval");
mkdirSync(outDir, { recursive: true });
const splitName = safeReportNamePart(values.split!);
const file = path.join(outDir, `selection-${splitName}-${sample.length}-${Date.now()}.json`);
const meta = { split: values.split, n: sample.length, seed: Number(values.seed), promptVersion: SELECTION_PROMPT_VERSION, createdAt: new Date().toISOString() };
writeFileSync(file, JSON.stringify({ meta, models: report }, null, 2));
console.log(`report: ${file}`);
if (!values["no-import"]) {
const run = await importSelectBenchRun({ meta, models: report }, values.label ?? `${values.split} ${sample.length} 条 · ${Object.keys(report).join(" / ")}`, "script:eval-selection");
console.log(`SelectBench run: ${run.id}`);
}
await closeDb();