__tests__/local-prompt.test.js

Rama main · réplica de lectura del repositorio original.

/*
 * THE PROMPT, AND THE ONE BUG IN IT THAT COSTS A WHOLE TRAINING ROUND.
 *
 * Everything here guards a failure that produces a working-looking model and a bad one:
 *
 *   - the catalogue is missing, so the model must memorise the tool list and every new tool needs a
 *     training round before it can ever be chosen;
 *   - the catalogue describes a tool that is not the tool that runs, which is how `download_file`
 *     sat in the live catalogue for months pointing at dead code;
 *   - the numbered list is absent, so `click [13]` is a target derived from nothing and the model
 *     learns to guess indices — the exact mistake the fine-tune exists to remove;
 *   - a STALE numbered list is included, which is worse than none: it teaches the model to act on
 *     numbers that have already expired.
 */
import { describe, it, expect } from 'vitest';
import { systemFor, userFor, catalogue, signatureOf, purposeOf } from '../src/localPrompt.js';

const fn = (name, description, properties = {}, required = []) => ({
  type: 'function',
  function: { name, description, parameters: { type: 'object', properties, required } },
});

const TOOLS = [
  fn('look', 'Every clickable thing, numbered. The numbers are only valid until the page changes.'),
  fn('click', 'Click a numbered element on the page.', { index: { type: 'integer' }, why: { type: 'string' } }, ['index']),
  fn('type', 'Type into a numbered field.', { index: { type: 'integer' }, text: { type: 'string' } }, ['index', 'text']),
];

describe('the catalogue', () => {
  it('lists every tool with its arguments, so an unseen tool can still be chosen', () => {
    const c = catalogue(TOOLS);
    expect(c).toContain('look()');
    expect(c).toContain('click(index, why?)');
    /* Required arguments bare, optional marked — the shape of the answer reads left to right. */
    expect(c).toContain('type(index, text)');
  });

  it('keeps the LAST declaration when a name is declared twice', () => {
    /* The live agent merges handlers with Object.assign, so the later declaration is the one that
       runs. A catalogue that documents the earlier one describes behaviour that does not exist —
       which is precisely how one tool in this codebase was unreachable for months while being
       advertised to the model on every single call. */
    const dupes = [...TOOLS, fn('click', 'Actually follows a link by url.', { url: { type: 'string' } }, ['url'])];
    const c = catalogue(dupes);
    expect(c).toContain('click(url)');
    expect(c).not.toContain('click(index, why?)');
    expect(c.match(/^click\(/gm)).toHaveLength(1);
  });

  it('clips a paragraph to one clause instead of paying for it on every example', () => {
    /* The live descriptions are paragraphs, and prompt length sets training time directly. */
    const long = fn('dig', 'Read the page properly. '.repeat(40));
    expect(purposeOf(long.function.description).length).toBeLessThanOrEqual(90);
  });

  it('leaves a short description whole instead of eating its last word', () => {
    /* "Go to a web address." has no whitespace after the full stop, so a sentence search finds
       nothing; trimming a partial word off a string that was never truncated then produced
       "Go to a web" — every brief tool in the catalogue lost its object. */
    expect(purposeOf('Go to a web address.')).toBe('Go to a web address');
    expect(purposeOf('Go back to the previous page.')).toBe('Go back to the previous page');
  });

  it('cuts a long description at its first sentence, not mid-word', () => {
    const p = purposeOf('The CONTROLS on this page — buttons, links and fields — as a numbered list. More text here.');
    expect(p).toBe('The CONTROLS on this page');
  });

  it('survives a tool with no description and one with no arguments', () => {
    expect(catalogue([fn('back', '')])).toBe('back()');
    expect(signatureOf({ name: 'look' })).toBe('look()');
  });
});

describe('the system message', () => {
  it('carries the catalogue and the answer shape', () => {
    const s = systemFor({ role: 'research.web', site: 'google', tools: TOOLS });
    expect(s).toContain('research.web');
    expect(s).toContain('google');
    expect(s).toContain('click(index, why?)');
    expect(s).toContain('{"tool":"<name>","args":{...}}');
  });

  it('puts the role playbook in the prompt, not in the adapter', () => {
    /* Playbooks are edited by hand between rounds. In the prompt a correction lands this afternoon;
       baked into an adapter it waits for a night of training on somebody's laptop. */
    const s = systemFor({ role: 'sales', tools: TOOLS, playbook: 'Never quote a price.' });
    expect(s).toContain('Never quote a price.');
  });

  it('is identical for identical inputs, because train and serve both call it', () => {
    const a = systemFor({ role: 'r', site: 's', tools: TOOLS });
    const b = systemFor({ role: 'r', site: 's', tools: TOOLS });
    expect(a).toBe(b);
  });
});

describe('the user message', () => {
  const marks = '[1] Sign in\n[2] Dodaj ofertę\n[3] Close';

  it('INCLUDES THE NUMBERED LIST, which is the only thing that makes an index learnable', () => {
    const u = userFor({ goal: 'Submit the offer', observed: [{ kind: 'look', text: 'useme.com — 3 things to click', marks }] });
    expect(u).toContain('[2] Dodaj ofertę');
  });

  it('includes only the MOST RECENT list, never an expired one', () => {
    /* Two lists in one prompt is worse than none. The older numbers no longer refer to anything on
       screen, and including them teaches exactly the stale-index click this is meant to prevent. */
    const u = userFor({
      goal: 'g',
      observed: [
        { kind: 'look', text: 'page one — 2 things', marks: '[1] Old thing\n[2] Older thing' },
        { kind: 'open', text: 'https://example.com/two' },
        { kind: 'look', text: 'page two — 3 things', marks },
      ],
    });
    expect(u).toContain('[2] Dodaj ofertę');
    expect(u).not.toContain('Old thing');
    /* The earlier look still appears as a line — it happened — just without its expired numbers. */
    expect(u).toContain('page one — 2 things');
  });

  it('says so plainly when nothing has been seen', () => {
    expect(userFor({ goal: 'Start' })).toContain('- nothing yet');
  });

  it('keeps the goal first, because it is the thing that does not change', () => {
    expect(userFor({ goal: 'Find three suppliers', observed: [] }).startsWith('GOAL: Find three suppliers')).toBe(true);
  });
});

describe('the student sees the page it decides on', () => {
  it('shows the page on the most recent observation that has one, and only that one', () => {
    /* The teacher was handed six thousand characters of the page; the student was told a page
       had been read. Two rounds scored zero on open, run_script and dig for exactly that. */
    const u = userFor({ goal: 'find the suppliers', observed: [
      { kind: 'read', text: 'read the page (900 characters)', content: 'OLD PAGE: nothing here' },
      { kind: 'scroll', text: 'scrolled down' },
      { kind: 'read', text: 'read the page (1200 characters)', content: 'NEW PAGE: Suppliers - Acme, Globex' },
    ] });
    expect(u).toMatch(/NEW PAGE: Suppliers/);
    expect(u).not.toMatch(/OLD PAGE/);
  });

  it('lets the numbered list win where a look came after the read', () => {
    /* A look step records only marks, and they are the page for a click. */
    const u = userFor({ goal: 'g', observed: [
      { kind: 'read', text: 'read the page (1200 characters)', content: 'PAGE TEXT' },
      { kind: 'look', text: 'Acme - 4 things to click', marks: '[1] Suppliers [2] Contact' },
    ] });
    expect(u).toMatch(/\[1\] Suppliers/);
    expect(u).not.toMatch(/PAGE TEXT/);
  });
});