__tests__/assistant.test.js

Rama main · réplica de lectura del repositorio original.

/**
 * The assistant: one chat, one agent. A message starts a harness turn with the reply exit; the answer
 * (via reply() or plain prose) lands as the assistant's turn with its steps and cards; a message while
 * a turn runs is spoken into it; the history is digested into the next turn's orientation; a turn
 * resumed after a restart re-attaches to its chat. Scripted model, temp dir, no network.
 */
import { describe, it, expect } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { OperatorRun } from '../src/operator/harness.js';
import { Registry } from '../src/operator/registry.js';
import { makeAssistant, REPLY_SPEC, labelOf } from '../src/operator/assistant.js';
import { assistantPrompt } from '../src/operator/assistantPrompt.js';

const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'as-'));
const call = (name, args) => ({ content: '', toolCalls: [{ name, args }] });
const prose = (content) => ({ content, toolCalls: [] });
const tick = () => new Promise((r) => setTimeout(r, 5));

/** A harness turn factory over a scripted model; every turn gets the same script unless a queue is given. */
function factory(scripts, seen) {
  const jobDir = tmp();
  return ({ goal, orientation, finishSpec, meta }) => {
    const turns = Array.isArray(scripts[0]) ? scripts.shift() : scripts;
    let i = 0;
    const chat = async ({ messages }) => { if (seen) seen.push(messages.map((m) => ({ ...m }))); const t = turns[Math.min(i, turns.length - 1)]; i++; return typeof t === 'function' ? t(messages) : t; };
    const reg = new Registry();
    reg.register('gb_watcher_feed', 'feed', { type: 'object', properties: { watcherId: { type: 'string' } } }, async ({ watcherId }) => ({ watcherId, items: [{ author: 'Ilya', state: 'waiting', draft: 'thanks Ilya' }] }));
    const run = new OperatorRun({ goal, chat, registry: reg, systemPrompt: 'sys', orientation, finishSpec, meta, persistDir: jobDir, startIterations: 20 });
    run.done = run.run();
    return run;
  };
}

describe('assistant', () => {
  it('answers through reply() with cards and the steps it took, and titles the chat from the first message', async () => {
    const seen = [];
    const a = makeAssistant({ dir: tmp(), startTurn: factory([
      call('gb_watcher_feed', { watcherId: 'facebook-post-watcher' }),
      call('reply', { text: '**Ilya** replied on your post — a draft is ready.', details: 'feed: 1 waiting', cards: [{ kind: 'results', watcherId: 'facebook-post-watcher', title: 'Open results' }] }),
    ], seen) });
    const c = a.create();
    expect(c.title).toBe('New chat');
    const r = a.send(c.id, 'do I have replies to react to?');
    expect(r.ok).toBe(true); expect(r.jobId).toMatch(/^op-/);
    expect(a.view(c.id).live).toBeTruthy();
    while (a.live.has(c.id)) await tick();
    const v = a.view(c.id);
    expect(v.title).toBe('do I have replies to react to?'); expect(v.live).toBeNull();
    expect(v.turns.map((t) => t.role)).toEqual(['user', 'assistant']);
    const t = v.turns[1];
    expect(t.text).toMatch(/Ilya/); expect(t.status).toBe('done'); expect(t.details).toBe('feed: 1 waiting');
    expect(t.cards[0]).toMatchObject({ kind: 'results', watcherId: 'facebook-post-watcher' });
    expect(t.steps.map((s) => s.name)).toEqual(['gb_watcher_feed']); expect(t.steps[0].label).toBe('Reading watcher results'); expect(t.steps[0].text).toBe('1 item · 1 waiting on you');   // a human line, never JSON
    // the model was told the goal and NOT asked for a task list first
    expect(seen[0][1].content).toMatch(/GOAL: do I have replies/); expect(seen[0][1].content).not.toMatch(/save_task_list first/);
    expect(a.list()[0]).toMatchObject({ id: c.id, turns: 2, running: false });
  });

  it('takes plain prose as the answer, digests the history into the next turn, and speaks into a running turn', async () => {
    const seen = [];
    let release; const gate = new Promise((r) => { release = r; });
    const a = makeAssistant({ dir: tmp(), startTurn: factory([
      [prose('Yes — two watchers run every 15 minutes.')],
      [async () => { await gate; return prose('Understood, keeping it short.'); }],
    ], seen) });
    const c = a.create('Ops');
    a.send(c.id, 'what runs right now?');
    while (a.live.has(c.id)) await tick();
    expect(a.view(c.id).turns[1].text).toBe('Yes — two watchers run every 15 minutes.');
    expect(a.view(c.id).turns[1].steps).toEqual([]);
    // second turn: history in the orientation; a word while it runs is spoken into it
    a.send(c.id, 'shorter please');
    const spoken = a.send(c.id, 'and no jargon');
    expect(spoken.spoken).toBe(true);
    release(); while (a.live.has(c.id)) await tick();
    const v = a.view(c.id);
    expect(v.turns.map((t) => t.role)).toEqual(['user', 'assistant', 'user', 'user', 'assistant']);
    expect(v.turns[3].spoken).toBe(true);
    const orient = seen[1][1].content;
    expect(orient).toMatch(/THE CONVERSATION SO FAR/); expect(orient).toMatch(/Owner: what runs right now\?/); expect(orient).toMatch(/You: Yes — two watchers/);
    expect(orient).toMatch(/Owner: shorter please/);
  });

  it('re-attaches a resumed turn by its chat id so the answer still lands', async () => {
    const dir = tmp();
    const a = makeAssistant({ dir, startTurn: factory([prose('never used')]) });
    const c = a.create('Resume');
    // a turn journal as a restart leaves it: the chat id in meta
    const reg = new Registry();
    const run = new OperatorRun({ goal: 'x', chat: async () => call('reply', { text: 'back after the restart' }), registry: reg, systemPrompt: 'sys', finishSpec: REPLY_SPEC, meta: { chatId: c.id }, persistDir: tmp() });
    run.done = run.run();
    expect(a.attach(run)).toBe(true);
    while (a.live.has(c.id)) await tick();
    expect(a.view(c.id).turns.at(-1)).toMatchObject({ role: 'assistant', text: 'back after the restart' });
    expect(a.attach({ meta: { chatId: 'nope' } })).toBe(false);
  });

  it('carries a picture a tool took onto its step and inlines it for the app', async () => {
    const shots = path.join(process.env.PROFILE_DIR || '/profiles', 'operator', 'shots');
    let file = null; try { fs.mkdirSync(shots, { recursive: true }); file = path.join(shots, 'test-' + Date.now() + '.jpg'); fs.writeFileSync(file, Buffer.from([0xff, 0xd8, 0xff, 0xd9])); } catch { file = null; }
    const a = makeAssistant({ dir: tmp(), startTurn: ({ goal, orientation, finishSpec, meta }) => {
      const reg = new Registry();
      // the real tools hand back JSON TEXT (clipped for the model), never an object — the picture must still be found
      reg.register('gb_look', 'look', { type: 'object', properties: {} }, async () => JSON.stringify({ url: 'https://x', title: 'Page', controls: [], text: '', screenshotUrl: file ? '/v1/operator/shots/' + path.basename(file) : null }));
      let i = 0; const chat = async () => (i++ === 0 ? call('gb_look', {}) : call('reply', { text: 'looked' }));
      const run = new OperatorRun({ goal, chat, registry: reg, systemPrompt: 'sys', orientation, finishSpec, meta, persistDir: tmp(), startIterations: 10 });
      run.done = run.run(); return run;
    } });
    const c = a.create('Look'); a.send(c.id, 'look at it');
    while (a.live.has(c.id)) await tick();
    const step = a.view(c.id).turns[1].steps[0];
    expect(step.label).toBe('Looking at the page'); expect(step.text).toBe('Page');
    if (file) { expect(step.image).toMatch(/\/v1\/operator\/shots\/test-/); expect(step.imageData).toMatch(/^data:image\/jpeg;base64,/); fs.unlinkSync(file); }
    else expect(step.image).toBeUndefined();   // no writable shots dir on this host: the step still lands
  });

  it('records what a person asked, not the recipe it wrote', () => {
    /*
     * The training set takes its prompt from job.goal. The assistant writes a plan, and storing
     * that as the goal taught the model to expect an ordered list with the url already in it.
     * Measured on a live run: the owner wrote "on ns.nl look up a train from Amsterdam to Eindhoven
     * around nine" and the stored goal was a twelve-clause recipe starting from a url that 404'd.
     */
    const p = assistantPrompt();
    /* Plain string containment, not regex: the instruction contains backticks and an apostrophe,
       and both have to survive being written down in a test. */
    expect(p.includes('ALWAYS PASS')).toBe(true);
    expect(p.includes('own words, verbatim')).toBe(true);
  });

  it('never hands a walk a path it invented', () => {
    /*
     * The prompt used to say "say the start url", and the assistant duly wrote one. Measured on a
     * live run: ns.nl/en/travellers/journeyplanner looked entirely plausible, returned a 404, and the
     * walk that followed it burnt 120 steps, 23 looks, a click loop and a stall without reporting an
     * answer. A walk can find a page. It cannot know that the url it was handed was imagined.
     *
     * Asserted on the text because that is where the behaviour lives — there is no code that builds
     * the goal, only this instruction, so an instruction quietly dropped is the whole regression.
     */
    const p = assistantPrompt();
    expect(p).toMatch(/NEVER write a path you have not actually seen/);
    expect(p).toMatch(/Name the domain and let the walk find/);
    expect(p).not.toMatch(/say the start url/);
  });

  it('labels steps for people and ships a prompt that carries the ladder and the operator method', () => {
    expect(labelOf('gb_watcher_run')).toBe('Running the watcher'); expect(labelOf('gb_zzz_thing')).toBe('zzz thing');
    const p = assistantPrompt();
    expect(p).toMatch(/IS THE ANSWER ALREADY GATHERED/); expect(p).toMatch(/gb_walk/); expect(p).toMatch(/CHANGE THE SMALLEST THING/);
    expect(p).not.toMatch(/You are THE GB OPERATOR/); expect(p).not.toMatch(/FINISH through the finish tool/);
    expect(REPLY_SPEC.map({ text: 'hi', cards: [{ kind: 'approvals' }] })).toMatchObject({ status: 'done', summary: 'hi', report: { answer: 'hi', cards: [{ kind: 'approvals' }] } });
  });
});