__tests__/training.test.js
Rama main · réplica de lectura del repositorio original.
/*
* THE STATUS SCREEN, AND THE FOUR WAYS A STATUS SCREEN QUIETLY LIES.
*
* This module exists so an unattended nightly loop is not indistinguishable from a broken one. That
* only holds if the screen is right when it matters, so every test here is one way it could be
* wrong while still looking fine:
*
* - a worse adapter reaches serving because the round that produced it was simply the most recent
* one to finish;
* - a device that died overnight still reads "training", because the round it never closed stays
* `running` for ever;
* - a round is offered for tonight over data that has not changed since the last one, costing a
* night of somebody's laptop to reproduce the same adapter;
* - the pipeline screen and the device hub disagree about the same device, because each worked the
* answer out for itself.
*
* None of these throw. They all produce a screen that is confidently wrong.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import fs from 'fs';
import os from 'os';
import path from 'path';
import training from '../src/training.js';
const { startRound, noteRound, checkRound, endRound, promote, current, byDevice, state } = training;
let dir;
beforeEach(() => {
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gb-training-'));
process.env.PROFILE_DIR = dir;
});
afterEach(() => {
delete process.env.PROFILE_DIR;
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* gone already */ }
});
const finished = (over = {}) => {
const r = startRound({ device: 'laptop', base: 'qwen2.5-0.5b', turns: 9000 });
return endRound(r.id, {
baseline: { agreement_pct: 41.2 },
result: { agreement_pct: 58.9 },
adapter: 'gb-role-lora-v1',
...over,
});
};
describe('rounds', () => {
it('records a round as running until the device says otherwise', () => {
const r = startRound({ device: 'laptop', base: 'qwen2.5-0.5b', turns: 100 });
expect(r.status).toBe('running');
expect(r.endedAt).toBe(null);
/* A round recorded at dispatch and never picked up looks identical to one that ran and
vanished. Telling those two apart is the entire job of this file. */
expect(training.allRounds().map((x) => x.id)).toEqual([r.id]);
});
it('keeps the device progress lines in order and bounded', () => {
const r = startRound({ device: 'laptop' });
for (let i = 0; i < 50; i++) noteRound(r.id, `step ${i}`);
const kept = training.allRounds()[0].lines;
expect(kept).toHaveLength(40);
expect(kept[kept.length - 1].text).toBe('step 49');
});
it('ignores progress for a round it has never heard of', () => {
expect(noteRound('r-nope', 'hello')).toBe(null);
});
it('keeps a round that made the model worse', () => {
/* The losing rounds are the evidence the gate works. Dropping them would leave a history in
which the model only ever improves, which is the history of a broken gate. */
const r = finished({ result: { agreement_pct: 22.0 } });
expect(r.status).toBe('done');
expect(training.allRounds()).toHaveLength(1);
});
});
describe('promotion', () => {
it('refuses a round that is still running', () => {
const r = startRound({ device: 'laptop' });
expect(promote(r.id).error).toMatch(/running/);
expect(current()).toBe(null);
});
it('refuses a round that was never measured', () => {
const r = startRound({ device: 'laptop' });
endRound(r.id, {});
expect(promote(r.id).error).toMatch(/no measurement/);
expect(current()).toBe(null);
});
it('REFUSES A ROUND THAT DID NOT BEAT ITS OWN BASELINE', () => {
/* The whole reason promotion is a separate act. A round that finished is not a round that won,
and the failure mode of conflating them is silent: the freshest adapter wins by being fresh,
the model gets worse, and every later round is measured against the worse model. */
const r = finished({ baseline: { agreement_pct: 58.0 }, result: { agreement_pct: 57.9 } });
expect(promote(r.id).error).toMatch(/did not beat the baseline/);
expect(current()).toBe(null);
});
it('refuses a tie, because a tie is not evidence', () => {
const r = finished({ baseline: { agreement_pct: 58.0 }, result: { agreement_pct: 58.0 } });
expect(promote(r.id).error).toMatch(/did not beat/);
});
it('REFUSES A MODEL THAT BEAT ITS BASELINE BY COLLAPSING', () => {
/*
* These are the real figures from the first round to finish this path. It scored 15.33%
* against a 3.33% baseline, so the one-number gate said yes - and it answered look to almost
* everything: scroll -> look, open -> look, read -> look, run_script -> look, finish -> look.
* look is correct on 10% of the paper and it said it on 36%.
*
* Every headline improved. Agreement rose because the cheap answer is also a common one, and
* unusable FELL from 21% to 16% because it had learnt to emit valid JSON. Nothing on the page
* said the model had stopped reading the page.
*
* Promoting it is the expensive mistake rather than the wasted night: rounds chain from
* whatever is serving, so a collapsed adapter becomes the starting weights for every round
* after it.
*/
const r = finished({
baseline: { agreement_pct: 3.33 },
result: {
agreement_pct: 15.33,
collapse: { tool: 'look', said_pct: 36.0, correct_pct: 10.0, ratio: 3.6, distinct: 8 },
},
});
const out = promote(r.id);
expect(out.error).toMatch(/collapsed onto look/);
expect(out.error).toMatch(/3\.6x/);
expect(current()).toBe(null);
});
it('promotes a win that answered each tool about as often as it was right', () => {
/* 1.0 is a model reading the page. The threshold is there for the collapse, not to police a
model that slightly over-reaches for a common tool. */
const r = finished({ result: { agreement_pct: 58.9, collapse: { tool: 'open', said_pct: 14.0, correct_pct: 14.0, ratio: 1.0, distinct: 22 } } });
expect(promote(r.id)).toMatchObject({ promoted: r.id });
});
it('does not refuse a round measured before anyone counted this', () => {
/* Absent is not refused. Locking out every earlier round on a number nobody took would be a
new bug wearing a gate's clothes. */
const r = finished({ result: { agreement_pct: 58.9 } });
expect(promote(r.id)).toMatchObject({ promoted: r.id });
});
it('promotes a measured win and records what it beat', () => {
const r = finished();
expect(promote(r.id)).toMatchObject({ promoted: r.id });
const live = current();
expect(live.adapter).toBe('gb-role-lora-v1');
expect(live.agreement).toBe(58.9);
/* What it beat travels with the pointer, so a rollback decision does not need the round list. */
expect(live.beat).toBe(41.2);
expect(live.device).toBe('laptop');
});
it('refuses a round it has never heard of', () => {
expect(promote('r-nope').error).toBe('no such round');
});
});
describe('what each device is doing', () => {
it('reports the latest round per device, not the latest running one', () => {
/* A round only becomes `done` because the device called endRound. A round killed by a closed
lid stays `running` for ever, so ranking `running` above `done` would let one dead round mask
every real round after it on that row — for ever, and silently. */
const dead = startRound({ device: 'laptop' });
expect(dead.status).toBe('running');
const later = finished();
const row = byDevice()['laptop'];
expect(row.roundId).toBe(later.id);
expect(row.status).toBe('done');
expect(row.result).toBe(58.9);
});
it('flags a running round that has gone silent rather than calling it progress', () => {
const r = startRound({ device: 'laptop' });
noteRound(r.id, 'epoch 1, step 200');
const row = byDevice(Date.now() + 91 * 60 * 1000)['laptop'];
expect(row.status).toBe('running');
expect(row.stale).toBe(true);
expect(row.silentForMin).toBe(91);
});
it('does not flag a round that is merely slow', () => {
const r = startRound({ device: 'laptop' });
noteRound(r.id, 'epoch 1, step 200');
const row = byDevice(Date.now() + 9 * 60 * 1000)['laptop'];
expect(row.stale).toBe(false);
});
it('carries the last thing the device said, which is the point of the row', () => {
const r = startRound({ device: 'laptop' });
noteRound(r.id, 'epoch 1, step 200');
noteRound(r.id, 'epoch 1, step 400, loss 0.71');
expect(byDevice()['laptop'].last.text).toBe('epoch 1, step 400, loss 0.71');
});
it('keeps devices apart', () => {
endRound(startRound({ device: 'laptop-a' }).id, { baseline: { agreement_pct: 1 }, result: { agreement_pct: 2 } });
startRound({ device: 'laptop-b' });
const rows = byDevice();
expect(rows['laptop-a'].status).toBe('done');
expect(rows['laptop-b'].status).toBe('running');
});
});
describe('the one answer both screens read', () => {
/* Shaped as the tally hands it over, because that is what the endpoint passes in. */
const corpusOf = (ids) => ({
tiers: Object.values(ids).reduce((acc, v) => {
if (v && v.tier) acc[v.tier] = (acc[v.tier] || 0) + 1;
return acc;
}, { gold: 0, silver: 0, bronze: 0, void: 0 }),
ids,
unjudged: 0,
jobs: Object.keys(ids).length,
});
it('counts only usable work recorded since the last round', () => {
const r = finished();
const after = new Date(Date.parse(r.startedAt) + 60000).toISOString();
const before = new Date(Date.parse(r.startedAt) - 60000).toISOString();
const s = state({
corpus: corpusOf({
a: { tier: 'gold', at: after }, b: { tier: 'silver', at: after },
/* Bronze and void are evidence of what NOT to do; they do not make a round worth running. */
c: { tier: 'bronze', at: after }, d: { tier: 'void', at: after },
e: { tier: 'gold', at: before },
}),
});
expect(s.corpus.tiers).toEqual({ gold: 2, silver: 1, bronze: 1, void: 1 });
expect(s.corpus.usableSinceLastRound).toBe(2);
/* Two new jobs is not a night of somebody's laptop. */
expect(s.corpus.enoughForARound).toBe(false);
});
it('says yes the first time, when there is no last round to compare against', () => {
const s = state({ corpus: corpusOf({ a: { tier: 'gold', at: new Date().toISOString() } }) });
expect(s.corpus.enoughForARound).toBe(true);
});
it('passes on that the history is still being read, rather than showing a number as final', () => {
const s = state({ corpus: { ...corpusOf({}), scanning: true, pending: 1800 } });
expect(s.corpus.scanning).toBe(true);
expect(s.corpus.pending).toBe(1800);
});
it('hands the device hub the same rows as the pipeline screen', () => {
/* Both screens read one answer. Deriving it twice is how two screens end up disagreeing about
the same device in front of the person who has to trust them. */
const r = startRound({ device: 'laptop' });
noteRound(r.id, 'epoch 1');
const s = state({ trainers: [{ name: 'laptop', online: true }, { name: 'phone', online: true }] });
expect(s.trainers[0].training.roundId).toBe(r.id);
expect(s.trainers[0].training).toEqual(s.byDevice['laptop']);
/* A trainer with no round says so, rather than being left off. */
expect(s.trainers[1].training).toBe(null);
});
it('survives an empty machine', () => {
const s = state({});
expect(s.corpus.jobsWithVerdict).toBe(0);
expect(s.rounds).toEqual([]);
expect(s.serving).toBe(null);
expect(s.trainers).toEqual([]);
});
});
describe('the recipe and the validation curve', () => {
/* A score with no recipe beside it cannot be reproduced, and a round with no curve cannot say
whether it was still learning when the clock stopped it. Both travel with the round. */
const recipe = { loraR: 32, loraScope: 'all', lr: 0.0002, epochs: 3, batch: 1, accum: 16, device: 'cpu', dtype: 'float32' };
it('keeps the recipe the device trained with', () => {
const r = startRound({ device: 'laptop', base: 'qwen', turns: 900, recipe });
expect(state({}).rounds.find((x) => x.id === r.id).recipe).toEqual(recipe);
});
it('records each validation point, in order, and tracks the best', () => {
const r = startRound({ device: 'laptop', base: 'qwen', turns: 900, recipe });
checkRound(r.id, { step: 10, turns: 160, valLoss: 1.42, trainLoss: 1.6, lr: 0.00019 });
checkRound(r.id, { step: 20, turns: 320, valLoss: 1.21, trainLoss: 1.3, lr: 0.00015 });
checkRound(r.id, { step: 30, turns: 480, valLoss: 1.25, trainLoss: 1.1, lr: 0.0001 });
const row = state({}).rounds.find((x) => x.id === r.id);
expect(row.validation.map((p) => p.valLoss)).toEqual([1.42, 1.21, 1.25]);
expect(row.bestValLoss).toBe(1.21);
expect(row.bestAtTurns).toBe(320);
});
it('ignores a point with no loss in it, and a round it has never heard of', () => {
const r = startRound({ device: 'laptop', base: 'qwen', turns: 900 });
checkRound(r.id, { step: 1, turns: 16 });
expect(checkRound('r-nope', { valLoss: 1 })).toBeNull();
const row = state({}).rounds.find((x) => x.id === r.id);
expect(row.validation).toEqual([]);
expect(row.bestValLoss).toBeNull();
expect(row.recipe).toBeNull();
});
it('does not let a recipe that is not an object in', () => {
const r = startRound({ device: 'laptop', base: 'qwen', turns: 900, recipe: 'r32' });
expect(state({}).rounds.find((x) => x.id === r.id).recipe).toBeNull();
});
});
describe('a baseline is a number under conditions', () => {
const training = require('../src/training');
it('the answer budget is part of the key', () => {
const at48 = training.baselineKey({ scope: 'base', base: '', paper: 'p1', turns: 354, answer: 48 });
const at320 = training.baselineKey({ scope: 'base', base: '', paper: 'p1', turns: 354, answer: 320 });
expect(at48).not.toBe(at320);
expect(at320).toContain('a320');
});
it('a key from before the budget existed keeps its old shape, so it is never handed back', () => {
const old = training.baselineKey({ scope: 'base', base: '', paper: 'p1', turns: 354 });
expect(old).not.toContain('|a');
expect(old).not.toBe(training.baselineKey({ scope: 'base', base: '', paper: 'p1', turns: 354, answer: 320 }));
});
it('a baseline belongs to a model, and the incumbent keeps the bare key', () => {
const inc = training.baselineKey({ scope: 'base', base: '', paper: 'p1', turns: 354, answer: 320, student: 'Qwen/Qwen2.5-0.5B-Instruct' });
const bare = training.baselineKey({ scope: 'base', base: '', paper: 'p1', turns: 354, answer: 320 });
const other = training.baselineKey({ scope: 'base', base: '', paper: 'p1', turns: 354, answer: 320, student: 'Qwen/Qwen3-0.6B' });
/* Nothing already measured moves: the incumbent's key is the one it always had. */
expect(inc).toBe(bare);
expect(other).not.toBe(bare);
expect(other).toContain('Qwen/Qwen3-0.6B');
});
it('everything else still separates two numbers', () => {
const a = training.baselineKey({ scope: 'base', base: 'hub:r-1', paper: 'p1', turns: 354, answer: 320 });
const b = training.baselineKey({ scope: 'platform:google', base: 'hub:r-1', paper: 'p1', turns: 354, answer: 320 });
const c = training.baselineKey({ scope: 'base', base: 'hub:r-2', paper: 'p1', turns: 354, answer: 320 });
expect(new Set([a, b, c]).size).toBe(3);
});
});
describe('an adapter belongs to the model it was trained on', () => {
const fs = require('fs');
const os = require('os');
const path = require('path');
let dir, training;
beforeEach(() => {
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gb-student-of-'));
process.env.PROFILE_DIR = dir;
delete require.cache[require.resolve('../src/training')];
training = require('../src/training');
fs.mkdirSync(path.join(dir, 'training'), { recursive: true });
fs.writeFileSync(path.join(dir, 'training', 'rounds.json'), JSON.stringify([
{ id: 'r-small', status: 'done', recipe: { base: 'Qwen/Qwen2.5-0.5B-Instruct' }, adapterHub: 'hub:r-small', adapterLast: 'hub:r-small-last' },
{ id: 'r-big', status: 'done', recipe: { base: 'Qwen/Qwen3-1.7B' }, adapterHub: 'hub:r-big' },
]));
});
afterEach(() => {
delete process.env.PROFILE_DIR;
delete require.cache[require.resolve('../src/training')];
fs.rmSync(dir, { recursive: true, force: true });
});
it('names the model behind an adapter, by round id or by hub name', () => {
expect(training.studentOf('hub:r-small')).toBe('Qwen/Qwen2.5-0.5B-Instruct');
expect(training.studentOf('r-small')).toBe('Qwen/Qwen2.5-0.5B-Instruct');
expect(training.studentOf('hub:r-big')).toBe('Qwen/Qwen3-1.7B');
});
it('a mid-round checkpoint belongs to the same model as its round', () => {
expect(training.studentOf('hub:r-small-last')).toBe('Qwen/Qwen2.5-0.5B-Instruct');
});
it('says nothing rather than guessing when it cannot tell', () => {
expect(training.studentOf('hub:r-nobody')).toBe('');
expect(training.studentOf('')).toBe('');
});
});
describe('a different student is measured, not promoted by itself', () => {
const fs = require('fs');
const os = require('os');
const path = require('path');
let dir, training;
const paper = 'p-1';
const write = (rows, current) => {
fs.mkdirSync(path.join(dir, 'training'), { recursive: true });
fs.writeFileSync(path.join(dir, 'training', 'rounds.json'), JSON.stringify(rows));
fs.writeFileSync(path.join(dir, 'training', 'current.json'), JSON.stringify(current));
};
const round = (id, model, pct, extra = {}) => ({
id, status: 'done', device: 'Modal L4', paper,
scope: { level: 'base', name: '', key: 'base' },
recipe: { base: model },
baseline: { agreement_pct: 6.2, turns: 354 },
result: { agreement_pct: pct, turns: 354, collapse: { ratio: 1.2, tool: 'read', distinct: 20 } },
adapterHub: `hub:${id}`, adapter: `hub:${id}`,
...extra,
});
beforeEach(() => {
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gb-promote-'));
process.env.PROFILE_DIR = dir;
delete require.cache[require.resolve('../src/training')];
training = require('../src/training');
});
afterEach(() => {
delete process.env.PROFILE_DIR;
delete require.cache[require.resolve('../src/training')];
fs.rmSync(dir, { recursive: true, force: true });
});
it('refuses the automatic promotion of a round on another model, and says whose scope it is', () => {
write([
round('r-new', 'Qwen/Qwen3-1.7B', 44.0),
round('r-old', 'Qwen/Qwen2.5-0.5B-Instruct', 36.7, { promoted: true }),
], { adapter: 'hub:r-old', scopes: { base: { adapter: 'hub:r-old' } } });
const out = training.promote('r-new', { auto: true });
expect(out.error).toMatch(/not the student that serves/);
expect(out.error).toContain('Qwen/Qwen3-1.7B');
expect(out.error).toContain('Qwen/Qwen2.5-0.5B-Instruct');
});
it('but a person may still promote it by hand', () => {
write([
round('r-new', 'Qwen/Qwen3-1.7B', 44.0),
round('r-old', 'Qwen/Qwen2.5-0.5B-Instruct', 36.7, { promoted: true }),
], { adapter: 'hub:r-old', scopes: { base: { adapter: 'hub:r-old' } } });
const out = training.promote('r-new');
expect(String(out.error || '')).not.toMatch(/not the student that serves/);
});
it(`and the incumbent own rounds are untouched by the rule`, () => {
write([
round('r-next', 'Qwen/Qwen2.5-0.5B-Instruct', 40.0),
round('r-old', 'Qwen/Qwen2.5-0.5B-Instruct', 36.7, { promoted: true }),
], { adapter: 'hub:r-old', scopes: { base: { adapter: 'hub:r-old' } } });
const out = training.promote('r-next', { auto: true });
expect(String(out.error || '')).not.toMatch(/not the student that serves/);
});
it('says a hold is a hold, so the caller does not throw the round away', () => {
/*
* The round that found this: Qwen3 scored 35.31% against the serving 36.72% on tool choice
* and 20.06% against 19.21% on whole calls, and was discarded by the rule meant to protect
* it. Refused-is-discarded is right for a failure and wrong for a decision waiting on a person.
*/
write([
round('r-new', 'Qwen/Qwen3-0.6B', 35.31),
round('r-old', 'Qwen/Qwen2.5-0.5B-Instruct', 36.72, { promoted: true }),
], { adapter: 'hub:r-old', scopes: { base: { adapter: 'hub:r-old' } } });
const out = training.promote('r-new', { auto: true });
expect(out.hold).toBe(true);
expect(out.error).toMatch(/promote by hand/);
});
it('and a real failure carries no hold, so it is discarded as before', () => {
write([
round('r-worse', 'Qwen/Qwen2.5-0.5B-Instruct', 4.0),
round('r-old', 'Qwen/Qwen2.5-0.5B-Instruct', 36.72, { promoted: true }),
], { adapter: 'hub:r-old', scopes: { base: { adapter: 'hub:r-old' } } });
const out = training.promote('r-worse', { auto: true });
expect(out.error).toBeTruthy();
expect(out.hold).toBeFalsy();
});
});