__tests__/training-plan.test.js
Rama main · réplica de lectura del repositorio original.
/*
* THE DECISION TO TRAIN, AND THE WAYS AN AUTOMATIC LOOP QUIETLY STOPS BEING ONE.
*
* None of these throw. Each produces a pipeline that looks healthy and does nothing, or one that
* does the wrong thing enthusiastically:
*
* - the gate waits for "new data" while 98% of the set it already has sits unlearned;
* - a round that segfaulted holds the slot for ever, so no round ever starts again;
* - coverage is summed from turns AVAILABLE rather than turns TRAINED, so the corpus reads as
* finished after one night;
* - a losing round becomes the base for the next one, and the damage compounds;
* - the owner's switch is overridden by a rule that judged the reasons good.
*/
import { describe, it, expect } from 'vitest';
import { decide, covered, ENOUGH_NEW } from '../src/trainingPlan.js';
const ready = (over = {}) => ({
corpus: { usableSinceLastRound: 0, scanning: false, pending: 0 },
dataset: { train: 23233 },
rounds: [],
trainers: [{ name: 'laptop-carla', deviceId: 'd1', online: true }],
auto: true,
serving: null,
...over,
});
describe('reasons to run', () => {
it('RUNS WHILE THE SET IS NOT COVERED, without waiting for new data', () => {
/* The reason that is easy to miss. The first round reached 685 of 23,233 turns; a gate that
only fires on "200 new runs" would leave the other 22,548 unlearned for ever while
reporting that everything was fine. */
const d = decide(ready({ rounds: [{ status: 'done', trained: 685 }] }));
expect(d.run).toBe(true);
expect(d.why).toContain('22548');
expect(d.coverage).toEqual({ seen: 685, total: 23233 });
});
it('runs on new data once the set IS covered', () => {
const d = decide(ready({
rounds: [{ status: 'done', trained: 23233 }],
corpus: { usableSinceLastRound: 240, scanning: false },
}));
expect(d.run).toBe(true);
expect(d.why).toContain('240 new usable');
});
it('waits when the set is covered and barely anything new has arrived', () => {
const d = decide(ready({
rounds: [{ status: 'done', trained: 23233 }],
corpus: { usableSinceLastRound: 12, scanning: false },
}));
expect(d.run).toBe(false);
expect(d.why).toContain(`wants ${ENOUGH_NEW}`);
});
it('counts turns TRAINED, not turns that were available', () => {
/* Each round records the whole set as available (23,233) and trains a few hundred. Summing the
wrong field reports the corpus as covered after a single night. */
expect(covered([{ trained: 685, turns: 23233 }, { trained: 700, turns: 23233 }])).toBe(1385);
/* A round that never reported what it trained contributes nothing rather than guessing. */
expect(covered([{ turns: 23233 }])).toBe(0);
});
});
describe('reasons not to run', () => {
it('never overrides the owner switch', () => {
const d = decide(ready({ auto: false, rounds: [{ status: 'done', trained: 0 }] }));
expect(d.run).toBe(false);
expect(d.why).toContain('switched off');
});
it('waits for a round that is genuinely running', () => {
const d = decide(ready({
rounds: [{ status: 'running', device: 'laptop-carla', startedAt: new Date().toISOString() }],
}));
expect(d.run).toBe(false);
expect(d.why).toContain('already running on laptop-carla');
});
it('DOES NOT LET A DEAD ROUND HOLD THE SLOT FOR EVER', () => {
/* Three rounds died with SIGSEGV while this was written. A crashed round never calls endRound,
so it stays `running` permanently — and without this clause the first crash would block every
round after it while the screen read "training". */
const longAgo = new Date(Date.now() - 4 * 3600 * 1000).toISOString();
const d = decide(ready({
rounds: [{ status: 'running', device: 'laptop-carla', startedAt: longAgo, lastAt: longAgo, trained: 0 }],
}));
expect(d.run).toBe(true);
});
it('will not start without a machine that can train', () => {
expect(decide(ready({ trainers: [] })).why).toContain('no machine');
expect(decide(ready({ trainers: [{ name: 'x', online: false }] })).why).toContain('no machine');
});
it('will not start without a set', () => {
expect(decide(ready({ dataset: null })).why).toContain('no training set');
});
it('will not decide on a corpus that is still being read', () => {
/* The history arrives in slices, so early on "not enough new data" is a statement about the
scan rather than about the data. */
const d = decide(ready({ corpus: { usableSinceLastRound: 0, scanning: true, pending: 1826 } }));
expect(d.run).toBe(false);
expect(d.why).toContain('1826');
});
});
describe('what a new round carries on from', () => {
it('CHAINS FROM WHAT IS SERVING, not from the last round that finished', () => {
/* A round that made the model worse is not promoted. Chaining from it anyway would push that
damage into every round after it; chaining from the serving adapter costs one round. */
const d = decide(ready({
rounds: [{ status: 'done', trained: 685, promoted: false }],
serving: { adapter: 'D:/gb-train/rounds/round-A/adapter' },
}));
expect(d.base).toBe('D:/gb-train/rounds/round-A/adapter');
});
it('starts from the bare base when nothing has been promoted yet', () => {
expect(decide(ready({ rounds: [{ status: 'done', trained: 1 }] })).base).toBe('');
});
it('names the machine it picked, so the dispatch has somewhere to go', () => {
const d = decide(ready({ rounds: [{ status: 'done', trained: 1 }] }));
expect(d.device).toBe('laptop-carla');
expect(d.deviceId).toBe('d1');
});
});
describe('no round on painted cards', () => {
/*
* Two rounds trained on a set where 58% of the turns followed a read whose content was never
* recorded, and both collapsed onto look. A round now waits until the set holds at least a
* slice's worth of turns that can see the page they decide on.
*/
const ready = () => ({
corpus: { usableSinceLastRound: 0 }, dataset: { train: 21000 }, rounds: [],
trainers: [{ name: 'lap', deviceId: 'd1', online: true }], auto: true, serving: null,
});
it('waits while fewer sighted turns exist than a round draws, and says how many', () => {
const d = decide({ ...ready(), sighted: 218, sliceTurns: 3600 });
expect(d.run).toBe(false);
expect(d.why).toMatch(/only 218 turn\(s\)/);
expect(d.why).toMatch(/draws 3600/);
});
it('runs once the set can fill a slice with sighted turns', () => {
expect(decide({ ...ready(), sighted: 3600, sliceTurns: 3600 }).run).toBe(true);
});
it('treats a manifest that never counted as unknown, not as zero', () => {
/* An old manifest must not lock the loop shut on a number nobody took. */
expect(decide({ ...ready(), sighted: null, sliceTurns: 3600 }).run).toBe(true);
});
});
describe('a batch is closed once it has its machines', () => {
const { decide } = require('../src/trainingPlan');
const now = new Date().toISOString();
const two = [{ name: 'KAROLINA', online: true }, { name: 'WOJMAGEMI', online: true }];
const s = [{ key: 'base', level: 'base', name: '', sighted: 2000, seen: 0, adapter: '', paper: 150 }];
const corpus = { usableSinceLastRound: 0, scanning: false };
it('does not hand a third share of a two-machine batch to the machine whose share finished', () => {
const rounds = [
{ id: 'r-a', status: 'done', device: 'WOJMAGEMI', startedAt: now, lastAt: now, endedAt: now, scope: { key: 'base' }, batch: 'r-a', share: 2, adapterHub: 'hub:r-a', result: 12, baseline: 5 },
{ id: 'r-b', status: 'running', device: 'KAROLINA', startedAt: now, lastAt: now, scope: { key: 'base' }, batch: 'r-a', share: 2 },
];
const d = decide({ corpus, dataset: { train: 12000 }, rounds, trainers: two, auto: true, serving: null, scopes: s, sliceTurns: 120, share: 2 });
expect(d.run).toBe(false);
expect(d.why).toMatch(/base is being trained/);
});
it('holds the scope while the merge round runs, and while it is pending', () => {
const rounds = [
{ id: 'r-a', status: 'done', device: 'WOJMAGEMI', startedAt: now, lastAt: now, endedAt: now, scope: { key: 'base' }, batch: 'r-a', share: 2, adapterHub: 'hub:r-a', mergedInto: 'r-m' },
{ id: 'r-b', status: 'done', device: 'KAROLINA', startedAt: now, lastAt: now, endedAt: now, scope: { key: 'base' }, batch: 'r-a', share: 2, adapterHub: 'hub:r-b', mergedInto: 'r-m' },
{ id: 'r-m', status: 'running', device: 'WOJMAGEMI', startedAt: now, lastAt: now, scope: { key: 'base' }, batch: 'r-a', share: 1, merge: true },
];
const d = decide({ corpus, dataset: { train: 12000 }, rounds, trainers: two, auto: true, serving: null, scopes: s, sliceTurns: 120, share: 2 });
expect(d.run).toBe(false);
const pend = [{ device: 'WOJMAGEMI', scope: { key: 'base', level: 'base', name: '' }, batch: 'r-a', share: 1, merge: true, at: now }];
const d2 = decide({ corpus, dataset: { train: 12000 }, rounds: rounds.slice(0, 2), trainers: two, auto: true, serving: null, scopes: s, sliceTurns: 120, share: 2, pending: pend });
expect(d2.run).toBe(false);
});
it('waits for the merge when every share is in and none has started', () => {
const rounds = [
{ id: 'r-a', status: 'done', device: 'WOJMAGEMI', startedAt: now, lastAt: now, endedAt: now, scope: { key: 'base' }, batch: 'r-a', share: 2, adapterHub: 'hub:r-a', mergedInto: 'pending' },
{ id: 'r-b', status: 'done', device: 'KAROLINA', startedAt: now, lastAt: now, endedAt: now, scope: { key: 'base' }, batch: 'r-a', share: 2, adapterHub: 'hub:r-b', mergedInto: 'pending' },
];
const d = decide({ corpus, dataset: { train: 12000 }, rounds, trainers: two, auto: true, serving: null, scopes: s, sliceTurns: 120, share: 2 });
expect(d.run).toBe(false);
});
it('opens the scope again once the merge round is done', () => {
const rounds = [
{ id: 'r-a', status: 'done', device: 'WOJMAGEMI', startedAt: now, lastAt: now, endedAt: now, scope: { key: 'base' }, batch: 'r-a', share: 2, adapterHub: 'hub:r-a', mergedInto: 'r-m' },
{ id: 'r-b', status: 'done', device: 'KAROLINA', startedAt: now, lastAt: now, endedAt: now, scope: { key: 'base' }, batch: 'r-a', share: 2, adapterHub: 'hub:r-b', mergedInto: 'r-m' },
{ id: 'r-m', status: 'done', device: 'WOJMAGEMI', startedAt: now, lastAt: now, endedAt: now, scope: { key: 'base' }, batch: 'r-a', share: 1, merge: true, result: 12, baseline: 5 },
];
const d = decide({ corpus, dataset: { train: 12000 }, rounds, trainers: two, auto: true, serving: null, scopes: s, sliceTurns: 120, share: 2 });
expect(d.run).toBe(true);
});
});
describe('the batch is cut to the hours at three epochs', () => {
const sizing = require('../src/sizing');
it('turns follow from hours and speed; the epochs never give', () => {
expect(sizing.EPOCHS).toBe(3);
/* 1.5 h at 87.5 s a turn: 1.5*3600*0.9 / (87.5*3) = 18.5 -> the floor of 20 */
expect(sizing.turnsFor({ hours: 1.5, secPerTurn: 87.5 })).toBe(20);
/* 12 h at 90 s: 12*3600*0.9/(90*3) = 144 */
expect(sizing.turnsFor({ hours: 12, secPerTurn: 90 })).toBe(144);
/* 3 h at 90 s: 36 */
expect(sizing.turnsFor({ hours: 3 })).toBe(36);
expect(sizing.turnsFor({ hours: 0 })).toBe(sizing.MIN_TURNS);
});
it('reads the speed off the trainer lines', () => {
expect(sizing.speedOf('at 87.5s a turn the hours allow 3 of 12 planned steps — schedule shortened')).toBe(87.5);
expect(sizing.speedOf('32/180 turn-passes · epoch 1/3 · step 2/3 (0/16 turns into the next) · loss 1.61 · lr 2.00e-04 · 48 min in, about 42 min left, 3.8 GB')).toBe(90);
expect(sizing.speedOf('3/180 turn-passes · epoch 1/3 · step 0/12 · loss 1.8 · 2 min in, about 88 min left')).toBe(null);
expect(sizing.speedOf('10/150 agreement 0.0% 0.06 turns/s')).toBe(null);
});
it('a machine gets its own speed, the batch the typical one, and the mode splits the hours', () => {
const rounds = [
{ id: 'r3', device: 'Karolina', secPerTurn: 120 },
{ id: 'r2', device: 'WojMagEmi', secPerTurn: 87.5 },
{ id: 'r1', device: 'WojMagEmi', secPerTurn: 200 },
];
expect(sizing.secPerTurnFor('wojmagemi', rounds)).toBe(87.5);
expect(sizing.secPerTurnFor('Third', rounds)).toBe(sizing.DEFAULT_SEC);
expect(sizing.typicalSpeed(rounds)).toBe(120);
const time = sizing.forSettings({ hours: 3, mode: 'time', share: 2, secPerTurn: 90 });
expect(time).toMatchObject({ hoursEach: 1.5, turnsEach: 20, batchTurns: 40, epochs: 3 });
const work = sizing.forSettings({ hours: 3, mode: 'work', share: 2, secPerTurn: 90 });
expect(work).toMatchObject({ hoursEach: 3, turnsEach: 36, batchTurns: 72 });
const long = sizing.forSettings({ hours: 12, mode: 'work', share: 2, secPerTurn: 90 });
expect(long.turnsEach).toBe(144);
});
it('a round remembers the speed its lines report', () => {
const fs = require('fs'); const os = require('os'); const path = require('path');
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gb-speed-'));
const prev = process.env.PROFILE_DIR; process.env.PROFILE_DIR = dir;
delete require.cache[require.resolve('../src/training')];
const training = require('../src/training');
try {
const r = training.startRound({ device: 'WojMagEmi', turns: 60 });
training.noteRound(r.id, '3/180 turn-passes · epoch 1/3 · step 0/12 · loss 1.8 · 2 min in, about 88 min left');
expect(training.allRounds().find((x) => x.id === r.id).secPerTurn).toBeUndefined();
training.noteRound(r.id, 'at 87.5s a turn the hours allow 3 of 12 planned steps — schedule shortened');
expect(training.allRounds().find((x) => x.id === r.id).secPerTurn).toBe(87.5);
expect(sizing.secPerTurnFor('WOJMAGEMI', training.allRounds())).toBe(87.5);
} finally {
if (prev === undefined) delete process.env.PROFILE_DIR; else process.env.PROFILE_DIR = prev;
delete require.cache[require.resolve('../src/training')];
}
});
});
describe('a GPU node is cut for a GPU until it has spoken', () => {
const sizing = require('../src/sizing');
it('takes the whole corpus on its first round', () => {
expect(sizing.secPerTurnFor('Modal T4', [], { gpu: true })).toBe(sizing.DEFAULT_GPU_SEC);
expect(sizing.turnsFor({ hours: 6, secPerTurn: sizing.DEFAULT_GPU_SEC })).toBeGreaterThan(3690);
expect(sizing.secPerTurnFor('Modal T4', [{ device: 'Modal T4', secPerTurn: 0.9 }], { gpu: true })).toBe(0.9);
expect(sizing.secPerTurnFor('Laptop', [], { gpu: false })).toBe(sizing.DEFAULT_SEC);
});
});
describe('which scope deserves an adapter', () => {
const { pickScope, SCOPE_FLOOR, PAPER_FLOOR } = require('../src/trainingPlan');
const base = { key: 'base', level: 'base', name: '', sighted: 4110, exam: 602, free: 0, adapter: 'hub:r-base' };
const rows = [
base,
{ key: 'platform:google', level: 'platform', name: 'google', sighted: 2491, exam: 355, free: 2491, adapter: '' },
{ key: 'platform:youtube', level: 'platform', name: 'youtube', sighted: 54, exam: 6, free: 54, adapter: '' },
{ key: 'platform:studio', level: 'platform', name: 'studio', sighted: 487, exam: 18, free: 487, adapter: '' },
{ key: 'role:research.reviews', level: 'role', name: 'research.reviews', sighted: 1120, exam: 201, free: 1120, adapter: '' },
{ key: 'role:reddit-draft-forhire-poster-v7', level: 'role', name: 'x', sighted: 19, exam: 0, free: 19, adapter: '' },
];
it('picks a platform on its own size and paper, whatever the machine could chew', () => {
/* A GPU slice of 4,400 turns used to exclude every scope in the map. */
const p = pickScope(rows, 4400);
expect(p.pick.key).toBe('platform:google');
expect(pickScope(rows, 55).pick.key).toBe('platform:google');
});
it('a scope whose paper is too small to believe is never trained', () => {
const thin = rows.filter((s) => s.key !== 'platform:google' && s.key !== 'role:research.reviews');
expect(pickScope(thin, 55).pick).toBe(null);
expect(PAPER_FLOOR).toBe(30);
expect(SCOPE_FLOOR).toBe(100);
});
it('platforms come before roles, and the biggest first', () => {
const p = pickScope(rows.filter((s) => s.key !== 'platform:google'), 55);
expect(p.pick.key).toBe('role:research.reviews');
});
it('a scope that already has its own adapter is not retrained while another has none', () => {
const served = rows.map((s) => (s.key === 'platform:google' ? { ...s, adapter: 'hub:r-g', free: 0 } : s));
expect(pickScope(served, 55).pick.key).toBe('role:research.reviews');
});
});
describe('the scope with the most to teach goes first', () => {
const { pickScope } = require('../src/trainingPlan');
const base = { key: 'base', level: 'base', name: '', sighted: 4110, exam: 602, adapter: 'hub:r-base' };
const google = { key: 'platform:google', level: 'platform', name: 'google', sighted: 2491, exam: 355, adapter: '' };
const facebook = { key: 'platform:facebook', level: 'platform', name: 'facebook', sighted: 222, exam: 35, adapter: '' };
const reviews = { key: 'role:research.reviews', level: 'role', name: 'research.reviews', sighted: 1120, exam: 201, adapter: '' };
it('base with 1,680 unseen turns beats a fresh platform with 222', () => {
const p = pickScope([{ ...base, free: 1680 }, { ...facebook, free: 222 }], 100);
expect(p.pick.key).toBe('base');
expect(p.why).toContain('1680');
});
it('but a fresh scope wins a near tie, having everything to gain', () => {
const p = pickScope([{ ...base, free: 1800 }, { ...google, free: 1500 }], 100);
expect(p.pick.key).toBe('platform:google');
expect(p.why).toContain('never seen');
});
it('and when base is spent the biggest fresh scope goes', () => {
const p = pickScope([{ ...base, free: 0 }, { ...facebook, free: 222 }, { ...reviews, free: 1120 }], 100);
expect(p.pick.key).toBe('role:research.reviews');
});
it('a refused scope still waits for new data', () => {
const p = pickScope([{ ...base, free: 0 }, { ...google, free: 2491, refused: true }], 100, { fresh: 3 });
expect(p.pick).toBe(null);
expect(p.why).toMatch(/refused at the gates/);
});
});
describe('a refusal is about one attempt', () => {
const { refusalStands, FRESH_LR } = require('../src/trainingPlan');
const refused = {
startedAt: '2026-09-25T14:00:00.000Z',
discarded: true,
promoted: false,
recipe: { lr: 2e-4, epochs: 3 },
};
it('bars a retry on the same set at the same rate', () => {
expect(refusalStands({ round: refused, builtAt: '2026-09-25T09:00:00.000Z', nextLr: FRESH_LR })).toBe(true);
});
it('lapses when the set was rebuilt after that round started', () => {
expect(refusalStands({ round: refused, builtAt: '2026-09-25T18:58:04.000Z', nextLr: FRESH_LR })).toBe(false);
});
it('lapses when the hub would no longer send that learning rate', () => {
expect(refusalStands({ round: refused, builtAt: '2026-09-25T09:00:00.000Z', nextLr: 4e-5 })).toBe(false);
});
it('holds when the rate moved only by rounding', () => {
expect(refusalStands({ round: refused, builtAt: '2026-09-25T09:00:00.000Z', nextLr: 2.01e-4 })).toBe(true);
});
it('lapses when the exam no longer cuts the answer off where it did', () => {
/* Tonight's base round: refused on an exam that scored `finish` and `note` as garbage. */
const old48 = { ...refused, recipe: { lr: 4e-5, answerTokens: 48 } };
expect(refusalStands({ round: old48, builtAt: '2026-09-25T09:00:00.000Z', nextLr: 4e-5, nextAnswer: 48 })).toBe(true);
expect(refusalStands({ round: old48, builtAt: '2026-09-25T09:00:00.000Z', nextLr: 4e-5, nextAnswer: 320 })).toBe(false);
});
it('a round from before the budget was recorded was measured under the old one', () => {
const noBudget = { ...refused, recipe: { lr: 4e-5 } };
expect(refusalStands({ round: noBudget, builtAt: '2026-09-25T09:00:00.000Z', nextLr: 4e-5, nextAnswer: 320 })).toBe(false);
/* And with nothing asked about the budget, the older rule stands unchanged. */
expect(refusalStands({ round: noBudget, builtAt: '2026-09-25T09:00:00.000Z', nextLr: 4e-5 })).toBe(true);
});
it('a promoted round is no refusal at all', () => {
expect(refusalStands({ round: { ...refused, promoted: true }, builtAt: '', nextLr: FRESH_LR })).toBe(false);
expect(refusalStands({ round: null })).toBe(false);
});
it('a round with no recipe is judged on its data alone', () => {
const bare = { startedAt: '2026-09-25T14:00:00.000Z', discarded: true, promoted: false };
expect(refusalStands({ round: bare, builtAt: '2026-09-25T09:00:00.000Z', nextLr: 4e-5 })).toBe(true);
expect(refusalStands({ round: bare, builtAt: '2026-09-25T18:58:04.000Z', nextLr: 4e-5 })).toBe(false);
});
});