test/account-dirty-retry.test.js
Rama master · réplica de lectura del repositorio original.
// accounts.json dirty bookkeeping — a FAILED persistence must keep the retry signal.
//
// Contract under test: `_dirty` means "memory holds state the on-disk file does not
// prove". Only a save that actually completed (durable temp write + publish) may
// clear it, and a save that failed must set it — including a save that started on a
// clean pool, and including the case where the rename published but the parent
// directory fsync could not prove the directory entry durable (fs-atomic marks that
// error `renamePublished`).
//
// Why it matters: `_dirty` is the ONLY retry trigger for lazy state (spend counters,
// rolling health window, cooldown deadlines). `flushDirty()` runs on a 30s timer and
// returns immediately when the flag is clear — so a writer that clears the flag before
// attempting the write converts one transient `EIO`/`ENOSPC` into "this batch sits in
// memory until some unrelated mutation happens to re-arm the flag", and the periodic
// flush can never recover it.
//
// Faults are injected at the real syscall boundary (node:fs wrapped, then re-published
// with syncBuiltinESMExports), scoped to this test's synthetic accounts.json temp file
// and its parent directory. `saveAccounts` / `saveAccountsSync` are never replaced, so
// the production composition — writeFileSyncDurable (temp write + fsync) →
// renameSyncWithRetry (rename + parent-dir fsync) — runs unchanged.
//
// Each case owns its own account key, so a case that fails early cannot change what a
// later case observes on disk.
import { after, before, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { createRequire, syncBuiltinESMExports } from 'node:module';
import { readFileSync, readdirSync, unlinkSync } from 'node:fs';
import { basename, join, resolve } from 'node:path';
import {
addAccountByKey, removeAccount, recordAccountSpend,
saveAccountsSync, __flushDirtyAccounts, __isAccountsDirty, __setModelCatalogDeps,
} from '../src/auth.js';
import { writeFileSyncDurable } from '../src/fs-atomic.js';
import { config } from '../src/config.js';
const DATA_DIR = resolve(config.sharedDataDir || config.dataDir);
const ACCOUNTS_FILE = join(DATA_DIR, 'accounts.json');
const EMAIL = 'state-dirty@example.invalid';
const KEY = n => `state-dirty-${n}$test`;
// ─── fs boundary: real functions, wrapped, scoped to this test's paths ───────
const fsCjs = createRequire(import.meta.url)('node:fs');
const real = {
openSync: fsCjs.openSync,
closeSync: fsCjs.closeSync,
writeSync: fsCjs.writeSync,
fsyncSync: fsCjs.fsyncSync,
renameSync: fsCjs.renameSync,
};
const fdPath = new Map();
const armed = { writeEio: false, dirFsyncEio: false };
const stats = { tempOpens: [], renames: [] };
const isAccountsTemp = p => typeof p === 'string' && p.startsWith(`${ACCOUNTS_FILE}.`) && p.endsWith('.tmp');
const isDataDir = p => typeof p === 'string' && resolve(p).toLowerCase() === DATA_DIR.toLowerCase();
const injectedEio = where => Object.assign(new Error(`injected EIO at ${where}`), {
code: 'EIO', errno: -4095, syscall: 'write',
});
function installFsBoundary() {
fsCjs.openSync = function (path, ...rest) {
const fd = real.openSync.call(this, path, ...rest);
fdPath.set(fd, String(path));
if (isAccountsTemp(path)) stats.tempOpens.push(basename(String(path)));
return fd;
};
fsCjs.closeSync = function (fd, ...rest) {
try { return real.closeSync.call(this, fd, ...rest); } finally { fdPath.delete(fd); }
};
fsCjs.writeSync = function (fd, ...rest) {
// One-shot: the injected failure belongs to the next temp-file write only.
if (armed.writeEio && isAccountsTemp(fdPath.get(fd))) { armed.writeEio = false; throw injectedEio('accounts temp write'); }
return real.writeSync.call(this, fd, ...rest);
};
fsCjs.fsyncSync = function (fd, ...rest) {
if (armed.dirFsyncEio && isDataDir(fdPath.get(fd))) { armed.dirFsyncEio = false; throw injectedEio('accounts dir fsync'); }
return real.fsyncSync.call(this, fd, ...rest);
};
fsCjs.renameSync = function (from, to, ...rest) {
if (isAccountsTemp(from)) stats.renames.push(basename(String(from)));
return real.renameSync.call(this, from, to, ...rest);
};
syncBuiltinESMExports();
}
function restoreFsBoundary() {
for (const [name, fn] of Object.entries(real)) fsCjs[name] = fn;
fdPath.clear();
syncBuiltinESMExports();
}
// ─── observation helpers (all read the real on-disk file) ───────────────────
function accountOnDisk(key) {
let rows;
try { rows = JSON.parse(readFileSync(ACCOUNTS_FILE, 'utf8')); } catch { return null; }
return Array.isArray(rows) ? rows.find(a => a.apiKey === key) ?? null : null;
}
const requestsOnDisk = key => accountOnDisk(key)?._totalSpend?.requests ?? 0;
const pendingTemps = () => readdirSync(DATA_DIR).filter(n => n.startsWith('accounts.json.') && n.endsWith('.tmp'));
const spend = (p, c, t) => ({ prompt_tokens: p, completion_tokens: c, total_tokens: t });
const created = [];
function createAccount(key) {
const account = addAccountByKey(key, EMAIL);
created.push(account);
assert.equal(accountOnDisk(key)?.apiKey, key, 'precondition: the fixture account is persisted');
return account;
}
before(() => {
// Offline: catalog sync is irrelevant here and must not fire real RPCs.
__setModelCatalogDeps({ disableConnectSync: true, getCascadeModelConfigs: async () => ({ configs: [] }) });
installFsBoundary();
// Seam self-check: with the patch installed but before syncBuiltinESMExports the
// real modules keep the pristine bindings and nothing would be injected — a probe
// that silently measures nothing. Prove the boundary reaches src/fs-atomic.js.
const selfCheckTarget = `${ACCOUNTS_FILE}.${process.pid}.selfcheck.tmp`;
let selfCheckCode = null;
armed.writeEio = true;
try { writeFileSyncDurable(selfCheckTarget, 'seam-self-check'); } catch (e) { selfCheckCode = e?.code ?? null; }
assert.equal(armed.writeEio, false, 'the fs-boundary injection was consumed at the real write');
assert.equal(selfCheckCode, 'EIO', 'the injection must reach src/fs-atomic.js ESM bindings');
try { unlinkSync(selfCheckTarget); } catch {}
stats.tempOpens.length = 0;
stats.renames.length = 0;
});
beforeEach(() => {
// Every case starts from a settled pool with a healthy boundary; a case that
// leaves dirt behind must not leak it into the next one.
armed.writeEio = false;
armed.dirFsyncEio = false;
__flushDirtyAccounts();
assert.equal(__isAccountsDirty(), false, 'precondition: the pool starts clean');
});
after(() => {
try { __flushDirtyAccounts(); } catch {}
for (const account of created) removeAccount(account.id);
__setModelCatalogDeps(null);
restoreFsBoundary();
});
describe('accounts.json dirty bookkeeping — persistence outcome owns the retry signal', () => {
it('keeps the pool dirty when the lazy flush write fails', () => {
createAccount(KEY(1));
recordAccountSpend(KEY(1), spend(5, 5, 10));
assert.equal(__isAccountsDirty(), true, 'a lazy mutation arms the flush');
armed.writeEio = true;
__flushDirtyAccounts();
assert.equal(armed.writeEio, false, 'the injected EIO was consumed by the real temp-file write');
assert.equal(__isAccountsDirty(), true, 'a failed save must leave the pool dirty for the next flush');
assert.equal(requestsOnDisk(KEY(1)), 0, 'the failed save published nothing');
assert.deepEqual(pendingTemps(), [], 'the failed temp file was cleaned up');
});
it('retries a failed batch on the next flush without waiting for another mutation', () => {
createAccount(KEY(2));
recordAccountSpend(KEY(2), spend(11, 4, 15));
const opensBefore = stats.tempOpens.length;
armed.writeEio = true;
__flushDirtyAccounts();
assert.equal(stats.tempOpens.length, opensBefore + 1, 'the flush attempted exactly one write');
assert.equal(requestsOnDisk(KEY(2)), 0, 'the failed attempt published nothing');
// Storage healthy again, NO new mutation: the periodic flush must recover the batch.
__flushDirtyAccounts();
assert.equal(requestsOnDisk(KEY(2)), 1, 'the pending batch reached disk on the next flush');
assert.equal(accountOnDisk(KEY(2))._totalSpend.totalTokens, 15, 'with the mutated values');
assert.equal(__isAccountsDirty(), false, 'the successful save cleared the flag');
// And a clean pool still writes nothing (the flag is not left permanently armed).
const opensAfterRetry = stats.tempOpens.length;
__flushDirtyAccounts();
assert.equal(stats.tempOpens.length, opensAfterRetry, 'a clean pool performs no further write');
});
it('marks the pool dirty when a direct save fails while previously clean', () => {
assert.equal(__isAccountsDirty(), false, 'precondition: clean pool, nothing pending');
armed.writeEio = true;
created.push(addAccountByKey(KEY(3), EMAIL)); // its own saveAccounts() is the direct write
assert.equal(armed.writeEio, false, 'the injected EIO was consumed by the real write');
assert.equal(__isAccountsDirty(), true, 'a direct save failure from a clean pool must leave it dirty');
assert.equal(accountOnDisk(KEY(3)), null, 'the failed save published nothing');
__flushDirtyAccounts();
assert.equal(accountOnDisk(KEY(3))?.apiKey, KEY(3), 'the retry persisted the in-memory account');
assert.equal(__isAccountsDirty(), false, 'and cleared the flag');
});
it('keeps the pool dirty when saveAccountsSync fails, and its retry persists', () => {
createAccount(KEY(4));
recordAccountSpend(KEY(4), spend(3, 2, 5));
assert.equal(__isAccountsDirty(), true, 'a lazy mutation armed the flag');
armed.writeEio = true;
saveAccountsSync();
assert.equal(armed.writeEio, false, 'the injected EIO was consumed by the real write');
assert.equal(__isAccountsDirty(), true, 'the sync flush must not consume the retry signal');
assert.equal(requestsOnDisk(KEY(4)), 0, 'the failed sync flush published nothing');
saveAccountsSync();
assert.equal(__isAccountsDirty(), false, 'a successful sync flush clears the flag');
assert.equal(requestsOnDisk(KEY(4)), 1, 'the retry landed on disk');
});
it('marks a clean pool dirty when a direct saveAccountsSync fails, and a later flush retries', () => {
createAccount(KEY(6));
assert.equal(__isAccountsDirty(), false, 'precondition: clean pool, nothing pending');
armed.writeEio = true;
saveAccountsSync(); // direct sync save of an already-clean pool
assert.equal(armed.writeEio, false, 'the injected EIO was consumed by the real write');
assert.equal(__isAccountsDirty(), true, 'a failed sync save from a clean pool must leave the pool dirty');
assert.equal(accountOnDisk(KEY(6))?.apiKey, KEY(6), 'the previously published account is untouched');
// No new mutation is needed: the ordinary dirty flush is the retry.
const renamesBefore = stats.renames.length;
__flushDirtyAccounts();
assert.equal(stats.renames.length, renamesBefore + 1, 'the retry published the current memory');
assert.equal(__isAccountsDirty(), false, 'and cleared the flag');
assert.equal(accountOnDisk(KEY(6))?.apiKey, KEY(6), 'the account survives the rewrite');
assert.deepEqual(pendingTemps(), []);
});
it('keeps the retry signal when the parent-directory fsync fails after a published rename', () => {
createAccount(KEY(5));
recordAccountSpend(KEY(5), spend(9, 1, 10));
const renamesBefore = stats.renames.length;
armed.dirFsyncEio = true;
__flushDirtyAccounts();
assert.equal(armed.dirFsyncEio, false, 'the directory-fsync EIO was consumed after a real rename');
assert.equal(requestsOnDisk(KEY(5)), 1, 'the rename had already published the new bytes');
assert.equal(__isAccountsDirty(), true, 'an unproven directory entry must not consume the retry signal');
assert.deepEqual(pendingTemps(), [], 'the consumed temp was renamed away, not left behind');
// The retry must publish from a FRESH temp file: the previous one was consumed
// by the successful rename, so re-renaming it is not an option.
__flushDirtyAccounts();
assert.equal(__isAccountsDirty(), false, 'the retry succeeded and cleared the flag');
const published = stats.renames.slice(renamesBefore);
assert.equal(published.length, 2, 'both attempts published');
assert.notEqual(published[0], published[1], 'the retry republished from a fresh temp file');
assert.equal(requestsOnDisk(KEY(5)), 1, 'the rewrite is idempotent — memory is not double-counted');
assert.deepEqual(pendingTemps(), []);
});
});