__tests__/operator-harness.test.js

Rama main · réplica de lectura del repositorio original.

/**
 * The operator harness, driven by a scripted model: the task list is state, the identical-call
 * breaker refuses a repeat and ends a run that keeps repeating, a run ends only through finish (prose
 * alone is nudged, then finished as blocked), the budget grows with done tasks and ends the run when
 * spent, the owner's `say` lands as the next turn, and the journal is written. No network, no disk
 * beyond a temp dir.
 */
import { describe, it, expect } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { OperatorRun } from '../src/operator/harness.js';
import { Registry } from '../src/operator/registry.js';

const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'op-'));
const scripted = (turns) => { let i = 0; return async ({ messages }) => { const t = turns[Math.min(i, turns.length - 1)]; i++; return typeof t === 'function' ? t(messages) : t; }; };
const call = (name, args) => ({ content: '', toolCalls: [{ name, args }] });
const say = (content) => ({ content, toolCalls: [] });

function baseRegistry() {
  const reg = new Registry();
  let reads = 0;
  reg.register('read_thing', 'read', { type: 'object', properties: { id: { type: 'string' } } }, async ({ id }) => ({ id, reads: ++reads, value: 'v' + id }));
  reg.register('wait_thing', 'wait', { type: 'object', properties: {} }, async () => ({ waited: true }), { repeatable: true });
  reg.register('fail_thing', 'fail', { type: 'object', properties: {} }, async () => ({ error: 'nope' }));
  return reg;
}

describe('OperatorRun', () => {
  it('runs tools, keeps the task list as state, and ends through finish with a report', async () => {
    const reg = baseRegistry();
    const chat = scripted([
      call('save_task_list', { tasks: [{ title: 'read' }, { title: 'prove' }] }),
      call('read_thing', { id: 'a' }),
      call('update_task', { index: 0, done: true, note: 'read a' }),
      call('finish', { status: 'done', summary: 'a is v-a', report: { evidence: 'read_thing returned va' } }),
    ]);
    const run = new OperatorRun({ goal: 'read a', chat, registry: reg, systemPrompt: 'sys', persistDir: tmp() });
    const v = await run.run();
    expect(v.status).toBe('done'); expect(v.finalLine).toMatch(/^DONE: a is v-a/); expect(v.report.evidence).toMatch(/va/);
    expect(v.tasks[0].done).toBe(true); expect(v.tasks[0].note).toBe('read a'); expect(v.tasks[1].done).toBe(false);
    expect(v.events.some((e) => e.kind === 'tool' && e.name === 'read_thing')).toBe(true);
  });

  it('refuses an identical non-repeatable call with a nudge, lets a repeatable one through, and stops a run that keeps repeating', async () => {
    const reg = baseRegistry();
    const chat = scripted([call('read_thing', { id: 'x' }), call('read_thing', { id: 'x' }), call('wait_thing', {}), call('wait_thing', {}), call('read_thing', { id: 'x' }), call('read_thing', { id: 'x' }), call('read_thing', { id: 'x' }), call('read_thing', { id: 'x' })]);
    const run = new OperatorRun({ goal: 'g', chat, registry: reg, systemPrompt: 'sys', persistDir: tmp() });
    const v = await run.run();
    const toolEvents = v.events.filter((e) => e.kind === 'tool');
    expect(toolEvents.filter((e) => e.name === 'read_thing').length).toBe(1);     // executed once, then refused
    expect(toolEvents.filter((e) => e.name === 'wait_thing').length).toBe(2);     // repeatable: both ran
    expect(v.status).toBe('blocked'); expect(v.finalLine).toMatch(/stuck repeating read_thing/);
    const refused = run.messages.filter((m) => m.role === 'tool' && /already called read_thing/.test(m.content));
    expect(refused.length).toBeGreaterThan(0);
  });

  it('nudges prose without tools, then finishes as blocked; a DONE: line in prose is honoured', async () => {
    const reg = baseRegistry();
    const v1 = await new OperatorRun({ goal: 'g', chat: scripted([say('thinking…'), say('still thinking'), say('and more'), say('hm')]), registry: reg, systemPrompt: 'sys', persistDir: tmp() }).run();
    expect(v1.status).toBe('blocked'); expect(v1.finalLine).toMatch(/without finishing/);
    const v2 = await new OperatorRun({ goal: 'g', chat: scripted([say('DONE: it works, verified')]), registry: reg, systemPrompt: 'sys', persistDir: tmp() }).run();
    expect(v2.status).toBe('done'); expect(v2.finalLine).toBe('DONE: it works, verified');
  });

  it('ends when the budget is spent and grows the budget as tasks get done', async () => {
    const reg = baseRegistry();
    let n = 0;
    const chat = async () => ({ content: '', toolCalls: [{ name: 'read_thing', args: { id: String(n++) } }] });
    const v = await new OperatorRun({ goal: 'g', chat, registry: reg, systemPrompt: 'sys', persistDir: tmp(), startIterations: 5, maxIterations: 8 }).run();
    expect(v.status).toBe('blocked'); expect(v.iterations).toBe(5); expect(v.finalLine).toMatch(/budget spent/);
    const reg2 = baseRegistry(); let m = 0;
    const chat2 = async () => (m === 0 ? (m++, call('save_task_list', { tasks: [{ title: 't' }] })) : m === 1 ? (m++, call('update_task', { index: 0, done: true })) : ({ content: '', toolCalls: [{ name: 'read_thing', args: { id: String(m++) } }] }));
    const v2 = await new OperatorRun({ goal: 'g', chat: chat2, registry: reg2, systemPrompt: 'sys', persistDir: tmp(), startIterations: 3, maxIterations: 10 }).run();
    expect(v2.iterations).toBe(10);   // 3 + 40 earned, capped at max 10
  });

  it('ends a run whose tools keep failing, and delivers the owner\'s say as the next user turn', async () => {
    const reg = baseRegistry(); let k = 0;
    const chat = async () => ({ content: '', toolCalls: [{ name: 'fail_thing', args: { n: k++ } }] });
    const v = await new OperatorRun({ goal: 'g', chat, registry: reg, systemPrompt: 'sys', persistDir: tmp() }).run();
    expect(v.status).toBe('blocked'); expect(v.finalLine).toMatch(/eight tool calls in a row failed/);
    const seen = [];
    const run = new OperatorRun({ goal: 'g', chat: scripted([(msgs) => { seen.push(msgs.map((x) => x.content)); run.say('look at Joe'); return call('read_thing', { id: '1' }); }, (msgs) => { seen.push(msgs.map((x) => x.content)); return call('finish', { status: 'done', summary: 'ok' }); }]), registry: reg, systemPrompt: 'sys', persistDir: tmp() });
    await run.run();
    expect(seen[1].some((c) => /The owner says: look at Joe/.test(c))).toBe(true);
  });

  it('journals the run to disk with the transcript for resume', async () => {
    const dir = tmp(); const reg = baseRegistry();
    const run = new OperatorRun({ goal: 'g', chat: scripted([call('finish', { status: 'done', summary: 'ok' })]), registry: reg, systemPrompt: 'sys', persistDir: dir });
    await run.run();
    const rec = JSON.parse(fs.readFileSync(path.join(dir, run.id + '.json'), 'utf8'));
    expect(rec.status).toBe('done'); expect(rec.messages.length).toBeGreaterThan(2); expect(rec.events.some((e) => e.kind === 'end')).toBe(true);
  });
});