scripts/harvestProposalPoliceEvals.ts
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env bun
import isBotUser from '@github/libs/isBotUser';
import isProposal from '@github/libs/isProposal';
/**
* Harvests real ProposalPolice cases out of Expensify/App into candidate fixtures for the offline
* evals in `evals/proposalPolice`.
*
* Run it with:
*
* bun ./scripts/harvestProposalPoliceEvals.ts
* bun ./scripts/harvestProposalPoliceEvals.ts --set=duplicateCheck --issues=40
* bun ./scripts/harvestProposalPoliceEvals.ts --set=commentIntent --issueNumbers=98504,78241
*
* It shells out to an authenticated `gh`, so no token plumbing is needed. `--issueNumbers` skips the
* search and walks exactly the issues named, which is how a case spotted by hand gets pulled in — the
* canonical bid-spam fixture lives on an issue that never carried the `Help Wanted` label, so no search
* this script runs would ever have found it.
*
* WHERE THE ORIGINAL TEXT COMES FROM
*
* The text these fixtures need is not in the REST API. When ProposalPolice withdraws a duplicate it
* *replaces* the comment body with the withdraw message, and when it flags a substantive edit it
* *prepends* a banner to the body, so `body` on a harvested comment is the bot's text rather than the
* contributor's.
*
* Both originals come back from the GraphQL `userContentEdits` connection on `IssueComment`, which was
* verified against live data before this script was written. Despite the field name, `diff` is **not** a
* unified diff: it is the **full comment body as of that revision**. Two more properties of the
* connection that this script relies on, also verified against live data:
*
* - Nodes come back newest-first.
* - The oldest node's `editedAt` equals the comment's `createdAt`, i.e. GitHub records the body the
* comment was created with as the first revision. A comment that was never edited has no nodes at
* all, so a comment with revisions always has at least two.
*
* So for a comment whose revision N is the bot's withdraw message, revision N+1 holds the proposal the
* contributor actually posted; and for a comment whose revision N carries the substantive-edit banner,
* revision N+1 is the body the contributor edited *to* and revision N+2 the body they edited *from*.
*
* RE-RUNNING
*
* Harvesting is separate from labelling, because the expected outcome on a fixture is a hand-applied
* judgement that a re-harvest must never clobber. This script only ever writes to
* `evals/proposalPolice/fixtures/harvested/`, which is gitignored. The labelled fixtures the evals
* actually read — `commentIntent.json`, `duplicateCheck.json`, `editCheck.json` — are checked in beside
* that directory and are edited by hand. After each run the script reports which harvested candidates
* are not yet represented in the labelled files, matched on `sourceHash`, so growing a fixture set is a
* matter of copying a candidate across, replacing `sourceUrl` with its `sourceHash`, and filling in the
* expected outcome.
*
* ANONYMIZATION
*
* This repository is public and the intent labels include SPAM, so a labelled fixture must never read as
* an accusation against a named contributor. Checked-in fixtures therefore carry no `sourceUrl` and no
* real GitHub comment IDs — the duplicate-check sets renumber theirs to small synthetic values, since
* nothing depends on the originals beyond matching `expectedDuplicateCommentID`.
*
* Author logins are replaced with obviously-fictional stand-ins. Assignment is per-fixture rather than
* global, so a login appearing in two fixtures can be `snorkmaiden` in one and `little-my` in the other.
* What matters is that same-author and different-author relationships survive *within* a fixture, because
* the duplicate check must never report a contributor's own earlier proposal. @-mentions inside bodies
* are rewritten the same way, and mentions of anyone not otherwise in the fixture become `@user`, so
* replaying a fixture can never ping a real person. Profile links, email addresses, and the login spelt
* into a link to the author's own fork are neutralized the same way, since contributors paste all three.
*
* The rest of a github.com URL — repository, issue number, permalink, line range — is left intact. That
* is the technical substance of a proposal, and blanking it would change what the model is being asked to
* judge.
*/
import {SUBSTANTIVE_EDIT_MESSAGE_REGEX} from '@prompts/proposalPolice/messages';
import CLI from 'expensify-common/CLI';
import {spawnSync} from 'node:child_process';
import {createHash} from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
const REPO = 'Expensify/App';
const FIXTURES_DIR = path.resolve(__dirname, '../evals/proposalPolice/fixtures');
const HARVEST_DIR = path.join(FIXTURES_DIR, 'harvested');
/** The bot's own text, matched against comment bodies to recognize the cases worth harvesting. */
const WITHDRAW_MESSAGE = '#### 🚫 Duplicated proposal withdrawn by 🤖 ProposalPolice.';
const EDIT_BANNER_PREFIX = '🚨 Edited by **proposal-police**:';
/** The notice the bot posts alongside a withdrawal, which names the proposal it considered the original. */
const DUPLICATE_NOTICE_REGEX = /Your proposal is a duplicate of an already \[existing proposal]\(https:\/\/github\.com\/Expensify\/App\/issues\/\d+#issuecomment-(\d+)\)/;
/** GitHub caps `search/issues` at 100 results per page. */
const SEARCH_PAGE_SIZE = 100;
/** GitHub caps connection page sizes at 100. */
const COMMENTS_PAGE_SIZE = 100;
/** Deep enough for the busiest proposal comments observed, which sit in the low tens of revisions. */
const EDITS_PAGE_SIZE = 50;
const COMMENTS_QUERY = `
query($owner: String!, $repo: String!, $number: Int!, $after: String) {
repository(owner: $owner, name: $repo) {
issue(number: $number) {
comments(first: ${COMMENTS_PAGE_SIZE}, after: $after) {
pageInfo { hasNextPage endCursor }
nodes {
databaseId
body
createdAt
author { login typename: __typename }
userContentEdits(first: ${EDITS_PAGE_SIZE}) { nodes { editedAt diff } }
}
}
}
}
}`;
type FixtureSetName = 'commentIntent' | 'duplicateCheck' | 'editCheck';
type Comment = {
commentID: number;
/** The body as it stands today, which for a harvested case is usually the bot's text. */
body: string;
/** Revision bodies, newest-first. Empty for a comment that was never edited. */
revisions: string[];
author: string;
isBot: boolean;
createdAt: string;
sourceUrl: string;
};
type CommentIntentCandidate = {
body: string;
id: string;
origin: 'harvested';
sourceUrl: string;
};
type DuplicateCheckProposal = {
author: string;
body: string;
commentID: number;
sourceUrl: string;
};
type DuplicateCheckCandidate = {
id: string;
newProposal: DuplicateCheckProposal;
origin: 'harvested';
priorProposals: DuplicateCheckProposal[];
/** The prior proposal ProposalPolice linked in its notice, i.e. what production decided. */
withdrawnAgainstCommentID: number | null;
};
type EditCheckCandidate = {
after: string;
before: string;
id: string;
origin: 'harvested';
sourceUrl: string;
};
/**
* Runs `gh` without a shell (so a search query can't be word-split or expanded) and parses its JSON.
*/
function gh<T>(args: string[]): T {
const result = spawnSync('gh', args, {encoding: 'utf8', maxBuffer: 64 * 1024 * 1024});
if (result.status !== 0) {
throw new Error(`gh ${args.join(' ')} failed: ${result.stderr || result.stdout}`);
}
const parsed: unknown = JSON.parse(result.stdout);
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the caller names the shape it asked GitHub for; a bad shape surfaces as an undefined field on a developer-run script, not in production.
return parsed as T;
}
/**
* Obviously-fictional stand-in logins, so nobody reading a fixture mistakes one for a real contributor.
* Assigned in order within a fixture; a fixture needing more authors than this wraps with a numeric suffix.
*/
const STAND_IN_LOGINS = ['snorkmaiden', 'hemulen', 'little-my', 'sniff', 'too-ticky', 'stinky', 'mymble', 'hodgkins', 'fillyjonk', 'whomper'];
/**
* Rewrites author logins to stand-in logins and neutralizes every @-mention in the given bodies, so a
* fixture carries the same author relationships as the real thread without naming anyone. The caller
* passes the logins whose identity the fixture depends on; anyone else mentioned in the text collapses
* to `@user`.
*/
function anonymize(logins: string[], bodies: string[]): {aliases: Map<string, string>; bodies: string[]} {
const aliases = new Map<string, string>();
for (const login of logins) {
if (aliases.has(login)) {
continue;
}
const index = aliases.size;
const suffix = index >= STAND_IN_LOGINS.length ? `-${Math.floor(index / STAND_IN_LOGINS.length)}` : '';
aliases.set(login, `${STAND_IN_LOGINS.at(index % STAND_IN_LOGINS.length)}${suffix}`);
}
const anonymizedBodies = bodies.map((body) => {
let anonymized = body
// GitHub logins are alphanumerics and hyphens, and a mention is only a mention at a word boundary.
.replaceAll(/(^|[^\w/])@([A-Za-z\d](?:[A-Za-z\d-]*[A-Za-z\d])?)/g, (match, prefix: string, login: string) => `${prefix}@${aliases.get(login) ?? 'user'}`)
// A github.com URL with a single path segment is a profile link, so it names someone the same way a mention does.
.replaceAll(/https:\/\/github\.com\/([A-Za-z\d](?:[A-Za-z\d-]*[A-Za-z\d])?)(?![\w/-])/g, (match, login: string) => `https://github.com/${aliases.get(login) ?? 'user'}`)
.replaceAll(/[\w.+-]+@[\w-]+\.[\w.]+/g, 'user@example.com');
// Contributors routinely link a branch on their own fork, which spells their login into a URL path
// (`github.com/<login>/App/tree/…`) or a compare ref (`compare/main...<login>:App:…`) rather than
// into a mention. Those are exactly the logins already in the alias map, so replace them wherever
// they appear as a whole word.
for (const [login, alias] of aliases) {
anonymized = anonymized.replaceAll(new RegExp(`(^|[^\\w-])${login}(?![\\w-])`, 'g'), `$1${alias}`);
}
return anonymized;
});
return {aliases, bodies: anonymizedBodies};
}
/**
* Fetches every comment on an issue along with its revision history.
*/
function fetchComments(issueNumber: number): Comment[] {
type CommentsPage = {
data: {
repository: {
issue: {
comments: {
pageInfo: {hasNextPage: boolean; endCursor: string | null};
nodes: Array<{
databaseId: number;
body: string;
createdAt: string;
author: {login: string; typename: string} | null;
userContentEdits: {nodes: Array<{editedAt: string; diff: string | null}>};
} | null>;
};
} | null;
};
};
};
const comments: Comment[] = [];
let after: string | null = null;
do {
const args = ['api', 'graphql', '-f', `query=${COMMENTS_QUERY}`, '-f', 'owner=Expensify', '-f', 'repo=App', '-F', `number=${issueNumber}`];
if (after) {
args.push('-f', `after=${after}`);
}
const page: CommentsPage = gh<CommentsPage>(args);
const connection = page.data.repository.issue?.comments;
if (!connection) {
break;
}
for (const node of connection.nodes) {
// A comment whose author deleted their account comes back with a null author and is unusable
// for a fixture, since same-author relationships can no longer be established for it.
if (!node?.author) {
continue;
}
comments.push({
commentID: node.databaseId,
body: node.body,
revisions: node.userContentEdits.nodes.map((edit) => edit.diff ?? ''),
author: node.author.login,
isBot: isBotUser(node.author.login, node.author.typename),
createdAt: node.createdAt,
sourceUrl: `https://github.com/${REPO}/issues/${issueNumber}#issuecomment-${node.databaseId}`,
});
}
after = connection.pageInfo.hasNextPage ? connection.pageInfo.endCursor : null;
} while (after);
return comments;
}
/**
* Searches issues by comment text and returns their numbers, newest first.
*/
function searchIssueNumbers(query: string, limit: number): number[] {
type SearchPage = {items: Array<{number: number}>};
const numbers: number[] = [];
for (let page = 1; numbers.length < limit; page++) {
const results = gh<SearchPage>([
'api',
'-X',
'GET',
'search/issues',
'-f',
`q=${query}`,
'-f',
'sort=created',
'-f',
'order=desc',
'-F',
`per_page=${Math.min(SEARCH_PAGE_SIZE, limit - numbers.length)}`,
'-F',
`page=${page}`,
]);
if (results.items.length === 0) {
break;
}
numbers.push(...results.items.map((item) => item.number));
}
return numbers.slice(0, limit);
}
/**
* Ordinary contributor comments, which is what the intent classifier sees in production. Comments the
* bot has touched are skipped: a withdrawn or bannered body is the bot's text, and a comment matching
* the template never reaches the classifier because the deterministic template check catches it first.
*/
function harvestCommentIntentCandidates(comments: Comment[]): CommentIntentCandidate[] {
const candidates: CommentIntentCandidate[] = [];
for (const comment of comments) {
if (comment.isBot || isProposal(comment.body) || comment.body.startsWith(WITHDRAW_MESSAGE) || comment.body.startsWith(EDIT_BANNER_PREFIX)) {
continue;
}
const {bodies} = anonymize([comment.author], [comment.body]);
candidates.push({
body: bodies.at(0) ?? '',
id: `commentIntent-${comment.commentID}`,
origin: 'harvested',
sourceUrl: comment.sourceUrl,
});
}
return candidates;
}
/**
* Withdrawn duplicates, reassembled into what the duplicate check saw: the proposal as posted, plus
* every proposal that predates it in the same issue.
*/
function harvestDuplicateCheckCandidates(comments: Comment[]): DuplicateCheckCandidate[] {
const candidates: DuplicateCheckCandidate[] = [];
for (const comment of comments) {
const withdrawnAt = comment.revisions.findIndex((revision) => revision.startsWith(WITHDRAW_MESSAGE));
const originalBody = withdrawnAt === -1 ? undefined : comment.revisions.at(withdrawnAt + 1);
if (!originalBody || !isProposal(originalBody)) {
continue;
}
// The bot's notice names the proposal it withdrew against, which is the outcome to compare an eval
// run to. It's posted right after the withdrawal and addresses the withdrawn proposal's author.
const notice = comments.find((other) => other.isBot && other.createdAt > comment.createdAt && other.body.includes(`@${comment.author} `) && DUPLICATE_NOTICE_REGEX.test(other.body));
const withdrawnAgainstCommentID = Number(DUPLICATE_NOTICE_REGEX.exec(notice?.body ?? '')?.at(1) ?? '') || null;
// Only proposals that existed when the bot ran were candidates, and each is taken as it read then
// rather than as it reads now, since a later withdrawal or edit would have rewritten its body.
const priors = comments
.filter((other) => other.commentID !== comment.commentID && other.createdAt < comment.createdAt && !other.isBot)
.map((other) => ({...other, body: other.revisions.at(-1) ?? other.body}))
.filter((other) => isProposal(other.body));
if (priors.length === 0) {
continue;
}
const {aliases, bodies} = anonymize([comment.author, ...priors.map((prior) => prior.author)], [originalBody, ...priors.map((prior) => prior.body)]);
candidates.push({
id: `duplicateCheck-${comment.commentID}`,
newProposal: {
author: aliases.get(comment.author) ?? '',
body: bodies.at(0) ?? '',
commentID: comment.commentID,
sourceUrl: comment.sourceUrl,
},
origin: 'harvested',
priorProposals: priors.map((prior, index) => ({
author: aliases.get(prior.author) ?? '',
body: bodies.at(index + 1) ?? '',
commentID: prior.commentID,
sourceUrl: prior.sourceUrl,
})),
withdrawnAgainstCommentID,
});
}
return candidates;
}
/**
* Real edits to proposals, as `(before, after)` pairs. Edits the bot flagged are recovered from the
* revision either side of the banner it prepended; edits it left alone are the plain revision pairs on
* a proposal comment it never touched.
*/
function harvestEditCheckCandidates(comments: Comment[]): EditCheckCandidate[] {
const candidates: EditCheckCandidate[] = [];
for (const comment of comments) {
const bannerAt = comment.revisions.findIndex((revision) => revision.startsWith(EDIT_BANNER_PREFIX));
const [before, after] = bannerAt === -1 ? [comment.revisions.at(1), comment.revisions.at(0)] : [comment.revisions.at(bannerAt + 2), comment.revisions.at(bannerAt + 1)];
// An earlier run's banner survives into later revisions, and the edit check is fed the proposal
// rather than the banner, so strip any that made it into the pair.
const beforeBody = before?.replace(SUBSTANTIVE_EDIT_MESSAGE_REGEX, '');
const afterBody = after?.replace(SUBSTANTIVE_EDIT_MESSAGE_REGEX, '');
if (!beforeBody || !afterBody || beforeBody === afterBody || !isProposal(beforeBody) || !isProposal(afterBody)) {
continue;
}
const {bodies} = anonymize([comment.author], [beforeBody, afterBody]);
candidates.push({
after: bodies.at(1) ?? '',
before: bodies.at(0) ?? '',
id: `editCheck-${comment.commentID}`,
origin: 'harvested',
sourceUrl: comment.sourceUrl,
});
}
return candidates;
}
/**
* The key a labelled fixture is matched on when reporting which candidates are new. Fixtures carry this
* rather than the source URL, so a checked-in file can't be read as a public accusation against a named
* contributor — this repository is public, and the intent labels include SPAM.
*/
function sourceHash(sourceUrl: string): string {
return createHash('sha256').update(sourceUrl).digest('hex').slice(0, 12);
}
/**
* Writes a candidate file and reports how much of it is new against the hand-labelled fixture set.
*/
function writeCandidates(set: FixtureSetName, candidates: Array<{sourceUrl?: string; newProposal?: {sourceUrl: string}}>): void {
fs.mkdirSync(HARVEST_DIR, {recursive: true});
fs.writeFileSync(path.join(HARVEST_DIR, `${set}.json`), `${JSON.stringify(candidates, null, 4)}\n`);
const labelledPath = path.join(FIXTURES_DIR, `${set}.json`);
const parsedLabelled: unknown = fs.existsSync(labelledPath) ? JSON.parse(fs.readFileSync(labelledPath, 'utf8')) : [];
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- only sourceHash is read, and a fixture file missing it just reports the candidate as unlabelled.
const labelled = parsedLabelled as Array<{sourceHash?: string}>;
const labelledSourceHashes = new Set(labelled.map((fixture) => fixture.sourceHash));
const unlabelled = candidates.filter((candidate) => !labelledSourceHashes.has(sourceHash(candidate.sourceUrl ?? candidate.newProposal?.sourceUrl ?? '')));
console.log(`${set}: harvested ${candidates.length} candidates, ${labelled.length} already labelled in ${set}.json, ${unlabelled.length} not yet labelled.`);
console.log(` → ${path.relative(process.cwd(), path.join(HARVEST_DIR, `${set}.json`))}`);
}
const cli = new CLI({
namedArgs: {
issues: {
description: 'How many issues to walk per fixture set',
default: 25,
parse: (value: string) => Number(value),
},
issueNumbers: {
description: 'Comma-separated issue numbers to walk instead of searching, for pulling in a specific case by hand',
default: '',
supersedes: ['issues'],
},
set: {
description: 'Harvest only one of: commentIntent, duplicateCheck, editCheck',
default: 'all',
},
},
} as const);
const issueLimit = cli.namedArgs.issues;
const requestedSet = cli.namedArgs.set;
const explicitIssueNumbers = cli.namedArgs.issueNumbers
.split(',')
.filter(Boolean)
.map((issueNumber) => Number(issueNumber));
if (requestedSet === 'all' || requestedSet === 'commentIntent') {
// Sampled from Help Wanted issues rather than from anything the bot acted on, because the intent
// classifier's real population is ordinary discussion: almost none of it mentions "Proposal" at all.
//
// Note that the comments the bot answered with a template reminder are *not* a useful second
// population here: every one sampled so far already satisfies `isProposal`, since the reminder fires
// on proposals that the stale dashboard prompt judged against sections the template no longer has.
// The deterministic template check now catches all of those before the classifier ever runs.
const issueNumbers = explicitIssueNumbers.length > 0 ? explicitIssueNumbers : searchIssueNumbers(`repo:${REPO} is:issue label:"Help Wanted"`, issueLimit);
console.log(`commentIntent: walking ${issueNumbers.length} issues…`);
writeCandidates(
'commentIntent',
issueNumbers.flatMap((issueNumber) => harvestCommentIntentCandidates(fetchComments(issueNumber))),
);
}
if (requestedSet === 'all' || requestedSet === 'duplicateCheck') {
const issueNumbers = explicitIssueNumbers.length > 0 ? explicitIssueNumbers : searchIssueNumbers(`repo:${REPO} "${WITHDRAW_MESSAGE.replaceAll('#### ', '')}" in:comments`, issueLimit);
console.log(`duplicateCheck: walking ${issueNumbers.length} issues with a withdrawn duplicate…`);
writeCandidates(
'duplicateCheck',
issueNumbers.flatMap((issueNumber) => harvestDuplicateCheckCandidates(fetchComments(issueNumber))),
);
}
if (requestedSet === 'all' || requestedSet === 'editCheck') {
const issueNumbers = explicitIssueNumbers.length > 0 ? explicitIssueNumbers : searchIssueNumbers(`repo:${REPO} "${EDIT_BANNER_PREFIX}" in:comments`, issueLimit);
console.log(`editCheck: walking ${issueNumbers.length} issues with a flagged edit…`);
writeCandidates(
'editCheck',
issueNumbers.flatMap((issueNumber) => harvestEditCheckCandidates(fetchComments(issueNumber))),
);
}