scripts/verify-pi-tool-scope.ts
Rama main · réplica de lectura del repositorio original.
// Official Pi CLI, disposable HOME, synthetic loopback provider and MCP tools.
// This verifies the CLI contract; it is not a real-local-model benchmark.
import assert from "node:assert/strict";
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { createServer } from "node:http";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
const cli = resolve(process.argv[2] ?? "node_modules/.bin/pi");
const home = mkdtempSync(join(tmpdir(), "omb-pi-selection-"));
process.env.HOME = home;
process.env.USERPROFILE = home;
process.env.OMB_DATA_DIR = join(home, "data");
const { PiDriver } = await import("../server/drivers/pi.ts");
const { ensureDirs } = await import("../server/config.ts");
if (process.argv[3]) {
const { SPAWNED_PROXIES } = await import("../server/proxy-paths.ts");
const directory = join(home, "package", "drivers"); mkdirSync(directory, { recursive: true });
const isolatedExtension = join(directory, "pi-mcp-extension.ts");
copyFileSync(resolve(process.argv[3]), isolatedExtension);
SPAWNED_PROXIES.piMcpExtension = isolatedExtension;
}
ensureDirs();
type Body = { tools?: Array<{ function: { name: string } }>; messages: Array<{ role: string; content?: unknown }> };
const payloads: Body[] = [];
let scenario = "native";
let round = 0;
const artifact = join(home, "draft.txt");
const blocked = join(home, "blocked.txt");
const receipt = join(home, "mail-calls.txt");
const provider = createServer(async (req, res) => {
if (!req.url?.endsWith("/chat/completions")) { res.writeHead(404).end(); return; }
let body = "";
for await (const chunk of req) body += chunk;
const payload = JSON.parse(body) as Body;
payloads.push(payload);
const calls = scenario === "native" ? [
{ name: "write", arguments: JSON.stringify({ path: artifact, content: "verified drafting" }) },
{ name: "bash", arguments: JSON.stringify({ command: `touch '${blocked}'` }) },
] : scenario === "mail" ? [
{ name: "mail_read_notes", arguments: "{}" }, { name: "mail_write_notes", arguments: "{}" },
] : [];
const toolCalls = round++ === 0 && calls.length > 0;
const delta = toolCalls ? { role: "assistant", tool_calls: calls.map((fn, index) => ({ index, id: `fixture-${index}`, type: "function", function: fn })) } : { role: "assistant", content: "Fixture finished." };
const frame = (change: unknown, finish_reason: string | null) => `data: ${JSON.stringify({ id: "fixture", object: "chat.completion.chunk", created: 0, model: "fixture", choices: [{ index: 0, delta: change, finish_reason }] })}\n\n`;
res.writeHead(200, { "content-type": "text/event-stream" });
res.end(frame(delta, null) + frame({}, toolCalls ? "tool_calls" : "stop") + "data: [DONE]\n\n");
});
await new Promise<void>((done) => provider.listen(0, "127.0.0.1", done));
const port = (provider.address() as { port: number }).port;
const agentDir = join(home, ".pi", "agent"); mkdirSync(agentDir, { recursive: true });
writeFileSync(join(agentDir, "models.json"), JSON.stringify({ providers: { fixture: {
baseUrl: `http://127.0.0.1:${port}/v1`, api: "openai-completions", apiKey: "synthetic-fixture-key",
models: [{ id: "fixture", name: "Synthetic contract fixture", reasoning: false, input: ["text"], contextWindow: 8192, maxTokens: 512, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } }],
} } }));
const packageExtension = join(home, "late-package.mjs");
writeFileSync(packageExtension, `import { Type } from "typebox";
import { writeFileSync } from "node:fs";
export default pi => {
pi.registerTool({name:"late_package",label:"Fixture package tool",description:"Must be withheld",parameters:Type.Object({}),
async execute(){writeFileSync(${JSON.stringify(blocked)},"package executed");return {content:[{type:"text",text:"unexpected"}],details:{}};}});
pi.on("before_agent_start",()=>pi.setActiveTools([...new Set([...pi.getActiveTools(),"bash","late_package"])]));
};`);
writeFileSync(join(agentDir, "settings.json"), JSON.stringify({ defaultProvider: "fixture", defaultModel: "fixture", defaultThinkingLevel: "off", extensions: [packageExtension] }));
const mail = join(home, "mail.mjs");
writeFileSync(mail, `import { createInterface } from "node:readline";
import { appendFileSync } from "node:fs";
createInterface({input:process.stdin}).on("line",line=>{
const m=JSON.parse(line); if(m.id===undefined)return; let result={};
if(m.method==="initialize")result={protocolVersion:m.params.protocolVersion,capabilities:{tools:{}},serverInfo:{name:"mail-fixture",version:"1"}};
if(m.method==="tools/list")result={tools:["read_notes","write_notes"].map(name=>({name,inputSchema:{type:"object",properties:{},additionalProperties:false}}))};
if(m.method==="tools/call"){appendFileSync(process.env.RECEIPT,m.params.name+"\\n");result={content:[{type:"text",text:"mail fixture receipt"}]};}
process.stdout.write(JSON.stringify({jsonrpc:"2.0",id:m.id,result})+"\\n");
});`);
const instance = await PiDriver.create({ instanceId: "fixture", displayName: "Pi contract fixture", enabled: true, config: { cli, fullAuto: false }, environment: { HOME: home, USERPROFILE: home } });
const events: any[] = [];
let approvals = 0;
instance.adapter.onEvent((event) => {
events.push(event);
if (event.type === "request.opened") {
approvals += 1;
void instance.adapter.respondToRequest(event.threadId, event.requestId, { behavior: "allow" });
}
});
const evidence: unknown[] = [];
async function turn(kind: string, toolScope: { allow: string[] }, resumeCursor?: string) {
scenario = kind; round = 0; const before = payloads.length; const eventStart = events.length;
const { turnId } = await instance.adapter.sendTurn({ threadId: "scope-contract", text: "Complete only the selected disposable fixture tools.", cwd: home, model: "fixture/fixture", approvalMode: "ask", toolScope, resumeCursor,
integrations: { custom: { mail: { command: process.execPath, args: [mail], env: { RECEIPT: receipt } } } },
});
const deadline = Date.now() + 30_000;
while (!events.slice(eventStart).some((event) => event.type === "turn.completed" && event.turnId === turnId) && Date.now() < deadline) await delay(50);
const completed = events.slice(eventStart).find((event) => event.type === "turn.completed" && event.turnId === turnId);
assert(completed?.ok === true, "Pi turn must complete successfully");
assert(payloads.length > before, JSON.stringify(events.slice(eventStart).filter((event) => event.type === "runtime.error")));
const names = payloads[before]!.tools?.map((tool) => tool.function.name) ?? [];
evidence.push({ scenario: kind, ok: completed.ok, names, schemaBytes: Buffer.byteLength(JSON.stringify(payloads[before]!.tools ?? [])), requests: payloads.length - before });
return { names, cursor: events.slice(eventStart).find((event) => event.type === "session.started")?.sessionId as string | undefined };
}
try {
const native = await turn("native", { allow: ["native:read", "native:edit", "native:write"] });
assert.deepEqual([...native.names].sort(), ["edit", "read", "write"]);
assert.equal(readFileSync(artifact, "utf8"), "verified drafting");
assert.equal(existsSync(blocked), false, "Withheld bash must never execute");
assert.deepEqual((await turn("empty", { allow: [] }, native.cursor)).names, []);
assert.deepEqual((await turn("unknown", { allow: ["native:unknown_tool"] })).names, []);
const selected = await turn("mail", { allow: ["mcp:mail:read_notes"] });
assert.deepEqual(selected.names, ["mail_read_notes"]);
assert.equal(readFileSync(receipt, "utf8"), "read_notes\n", "Only the raw selected MCP operation executes");
assert.equal(approvals, 1, "The selected custom tool still needs a human approval");
assert.deepEqual((await turn("empty", { allow: [] }, selected.cursor)).names, []);
console.log(JSON.stringify({ ok: true, cli, evidence, approvals, withheldNativeExecuted: false, withheldMcpExecuted: false }));
} finally {
await instance.dispose();
provider.closeAllConnections(); await new Promise<void>((done) => provider.close(() => done()));
rmSync(home, { recursive: true, force: true });
}