test/devin-connect-failover.test.js
Rama master · réplica de lectura del repositorio original.
import { afterEach, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
addAccountByKey, removeAccount, getAccountList, getApiKey, releaseAccount,
markRateLimited, getAccountAvailability, getAccountInternal,
__setReloginDeps, __resetReloginState,
} from '../src/auth.js';
import { classifyUpstreamError } from '../src/devin-connect.js';
import {
handleChatCompletions, __setConnectDeps, __resetConnectDeps,
} from '../src/handlers/chat.js';
// DEVIN_CONNECT cross-account failover: when a pooled account's session token is
// dead (UNAUTHORIZED) and same-account re-login can't revive it, the request
// must fall through to the NEXT healthy pooled account instead of failing.
// These exercise the real handler with the connect network calls + re-login
// mocked, so no token or socket is touched.
const createdIds = [];
const prevEnv = {};
function seed(label) {
const key = `devin-session-token$fo-${label}-${Math.random().toString(36).slice(2)}`;
const acct = addAccountByKey(key, label);
createdIds.push(acct.id);
return acct;
}
function unauthorized() {
return Object.assign(new Error('invalid token'), { code: 'UNAUTHORIZED' });
}
function transient() {
return Object.assign(new Error('upstream hiccup'), { status: 503 });
}
function capacity() {
// High-demand throttle as the upstream really sends it: a transient "model
// busy" wrapped in a 401 auth-shell, already classified to CAPACITY.
return Object.assign(new Error("We're currently facing high demand for this model. Please try again later."), { code: 'CAPACITY', status: 401 });
}
function fakeRes() {
const listeners = new Map();
return {
body: '', writableEnded: false,
write(chunk) { this.body += String(chunk); return true; },
end(chunk) { if (chunk) this.write(chunk); this.writableEnded = true; for (const cb of listeners.get('close') || []) cb(); },
on(event, cb) { if (!listeners.has(event)) listeners.set(event, []); listeners.get(event).push(cb); return this; },
};
}
function parseFrames(raw) {
return raw.split('\n\n').filter(Boolean).filter(f => !f.startsWith(':')).map(f => {
const d = f.split('\n').find(l => l.startsWith('data: '))?.slice(6) || '';
return d === '[DONE]' ? '[DONE]' : JSON.parse(d);
});
}
beforeEach(() => {
prevEnv.DEVIN_CONNECT = process.env.DEVIN_CONNECT;
prevEnv.MAX = process.env.DEVIN_CONNECT_FAILOVER_MAX;
prevEnv.RELOGIN = process.env.DEVIN_CONNECT_AUTO_RELOGIN;
process.env.DEVIN_CONNECT = '1';
delete process.env.DEVIN_CONNECT_FAILOVER_MAX;
delete process.env.DEVIN_CONNECT_AUTO_RELOGIN;
});
afterEach(() => {
__resetConnectDeps();
__resetReloginState();
__setReloginDeps(null);
if (prevEnv.DEVIN_CONNECT === undefined) delete process.env.DEVIN_CONNECT;
else process.env.DEVIN_CONNECT = prevEnv.DEVIN_CONNECT;
if (prevEnv.MAX === undefined) delete process.env.DEVIN_CONNECT_FAILOVER_MAX;
else process.env.DEVIN_CONNECT_FAILOVER_MAX = prevEnv.MAX;
if (prevEnv.RELOGIN === undefined) delete process.env.DEVIN_CONNECT_AUTO_RELOGIN;
else process.env.DEVIN_CONNECT_AUTO_RELOGIN = prevEnv.RELOGIN;
while (createdIds.length) removeAccount(createdIds.pop());
});
describe('DEVIN_CONNECT cross-account failover — non-stream', () => {
it('falls over to the next pooled account when the first token is dead and re-login fails', async () => {
const a = seed('dead-1');
const b = seed('healthy-2');
// Re-login can NOT recover (token-added accounts have no stored password).
__setReloginDeps({ windsurfLogin: async () => { throw new Error('no stored credential'); } });
const seen = [];
__setConnectDeps({
toChatCompletion: async (params) => {
seen.push(params.token);
if (params.token === a.apiKey) throw unauthorized();
return { status: 200, body: { id: 'x', choices: [{ message: { role: 'assistant', content: 'FAILOVER_OK' } }] } };
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 200);
assert.equal(result.body.choices[0].message.content, 'FAILOVER_OK');
// The dead account was tried first, then the healthy one — never re-picked.
assert.ok(seen.includes(a.apiKey), 'tried the dead account');
assert.ok(seen.includes(b.apiKey), 'failed over to the healthy account');
// No inflight leak: every account (dead + healthy) was finalized/released,
// so both are immediately re-selectable.
const reacquire = getApiKey([], null, '');
assert.ok(reacquire, 'pool is fully released after a failover hop');
releaseAccount(reacquire.apiKey);
});
it('prefers same-account re-login over failover when credentials exist', async () => {
const a = seed('relogin-1');
const originalToken = a.apiKey; // capture by value: reLoginAccount mutates a.apiKey
seed('other-2');
process.env.DEVIN_CONNECT_AUTO_RELOGIN = '1';
let freshToken = null;
__setReloginDeps({
isCredStoreEnabled: () => true,
getCredential: () => 'stored-password',
windsurfLogin: async () => {
freshToken = `devin-session-token$fresh-${Math.random().toString(36).slice(2)}`;
return { apiKey: freshToken, name: 'relogin-1' };
},
});
const seen = [];
__setConnectDeps({
toChatCompletion: async (params) => {
seen.push(params.token);
// Only the ORIGINAL token is dead; the re-logged-in fresh token works.
// (Compare to the captured value, not a.apiKey — re-login mutated it.)
if (params.token === originalToken) throw unauthorized();
return { status: 200, body: { id: 'x', choices: [{ message: { role: 'assistant', content: 'RELOGIN_OK' } }] } };
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 200);
assert.equal(result.body.choices[0].message.content, 'RELOGIN_OK');
// Recovery went through the SAME account's fresh token, not a second account.
assert.ok(freshToken && seen.includes(freshToken), 'retried on the re-logged-in token');
});
it('free→paid: fresh token still UNAUTHORIZED ⇒ MODEL_BLOCKED (no failover storm)', async () => {
// Live-fire finding (#42): a free account requesting a paid selector gets a
// bare `permission_denied`/"internal error" from upstream — byte-identical to
// a retired session token. The OLD code treated it as a dead token: re-login
// (mints a useless fresh token) → retry → fail over across the WHOLE pool →
// 401 "all accounts exhausted (dead session tokens)". That's a re-login storm
// against Auth1 (ban risk) plus a misleading error. The fix: a successful
// re-login proves the token was alive, so a 2nd UNAUTHORIZED on the fresh
// token is an entitlement wall → MODEL_BLOCKED (402), and NO failover.
const a = seed('paidwall-1');
const b = seed('paidwall-2');
const originalToken = a.apiKey;
process.env.DEVIN_CONNECT_AUTO_RELOGIN = '1';
__setReloginDeps({
isCredStoreEnabled: () => true,
getCredential: () => 'stored-password',
windsurfLogin: async () => ({ apiKey: `devin-session-token$fresh-${Math.random().toString(36).slice(2)}`, name: 'paidwall-1' }),
});
const seen = [];
__setConnectDeps({
// EVERY token (original, fresh, and the 2nd account) returns UNAUTHORIZED:
// that's what a paid model looks like to a free pool.
toChatCompletion: async (params) => { seen.push(params.token); throw unauthorized(); },
});
const result = await handleChatCompletions(
{ model: 'claude-opus-4.8', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 402, 'entitlement wall → 402, not 401');
assert.equal(result.body.error.code, 'MODEL_BLOCKED');
// Must NOT have failed over to the second account — that would multiply the
// re-login storm. Only the original + its one fresh-token retry were tried.
assert.ok(!seen.includes(b.apiKey), 'did not fail over to a second account');
assert.equal(seen.filter(t => t === originalToken).length, 1, 'original token tried exactly once');
});
it('returns 401 when every pooled account has a dead, unrecoverable token', async () => {
seed('dead-a');
seed('dead-b');
__setReloginDeps({ windsurfLogin: async () => { throw new Error('no credential'); } });
__setConnectDeps({ toChatCompletion: async () => { throw unauthorized(); } });
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 401);
assert.equal(result.body.error.code, 'UNAUTHORIZED');
});
it('honors DEVIN_CONNECT_FAILOVER_MAX=0 (no hop, just same-account re-login)', async () => {
const a = seed('nofail-1');
seed('nofail-2');
process.env.DEVIN_CONNECT_FAILOVER_MAX = '0';
__setReloginDeps({ windsurfLogin: async () => { throw new Error('no credential'); } });
const seen = [];
__setConnectDeps({
toChatCompletion: async (params) => { seen.push(params.token); throw unauthorized(); },
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 401);
// With max=0 only the first account is attempted — no failover hop.
assert.equal(seen.length, 1, 'no failover hop when max=0');
});
it('does NOT fail over on a non-account upstream error (CAPACITY surfaces directly)', async () => {
// CAPACITY = the MODEL is overloaded, not an account fault. Failing over would
// storm every account with the same doomed request, so it must surface as-is.
seed('err-1');
seed('err-2');
const seen = [];
__setConnectDeps({
toChatCompletion: async (params) => {
seen.push(params.token);
throw Object.assign(new Error('high demand'), { code: 'CAPACITY' });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 503);
assert.equal(seen.length, 1, 'CAPACITY (model overloaded) is not an account-failover trigger');
});
// R1: QUOTA_EXHAUSTED / RATE_LIMITED are ACCOUNT dry-wells. With a healthy
// account still in the pool, the request must fail over to it instead of
// surfacing 402/429 while that account sits idle. The old behavior surfaced the
// error on the first account — a real correctness bug (pool underutilized).
it('R1: fails over to a healthy account when the first hits RATE_LIMITED', async () => {
seed('rl-1');
seed('rl-2');
const seen = [];
let call = 0;
__setConnectDeps({
toChatCompletion: async (params) => {
seen.push(params.token);
if (call++ === 0) throw Object.assign(new Error('rate limited'), { code: 'RATE_LIMITED' });
return { status: 200, body: { id: 'x', choices: [{ message: { role: 'assistant', content: 'OK' } }] } };
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 200, 'served by the second, un-throttled account');
assert.equal(seen.length, 2, 'RATE_LIMITED on account 1 triggered one failover hop');
assert.notEqual(seen[0], seen[1], 'the hop landed on a different account');
});
it('R1: fails over to a healthy account when the first hits QUOTA_EXHAUSTED', async () => {
seed('q-1');
seed('q-2');
const seen = [];
let call = 0;
__setConnectDeps({
toChatCompletion: async (params) => {
seen.push(params.token);
if (call++ === 0) throw Object.assign(new Error('out of credit'), { code: 'QUOTA_EXHAUSTED' });
return { status: 200, body: { id: 'x', choices: [{ message: { role: 'assistant', content: 'OK' } }] } };
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 200, 'served by the second, funded account');
assert.equal(seen.length, 2, 'QUOTA_EXHAUSTED on account 1 triggered one failover hop');
});
it('R1: surfaces the account error once the pool is exhausted (all accounts dry)', async () => {
// Both accounts are quota-dry → after failover exhausts the pool, the client
// gets the real 402 rather than a hang or a misleading 401.
seed('qdry-1');
seed('qdry-2');
const seen = [];
__setConnectDeps({
toChatCompletion: async (params) => {
seen.push(params.token);
throw Object.assign(new Error('out of credit'), { code: 'QUOTA_EXHAUSTED' });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 402, 'exhausted pool surfaces the real QUOTA_EXHAUSTED status');
assert.equal(result.body.error.code, 'QUOTA_EXHAUSTED');
assert.equal(seen.length, 2, 'both accounts were tried before surfacing');
});
});
describe('DEVIN_CONNECT cross-account failover — stream', () => {
it('fails over before any byte is emitted, then streams from the healthy account', async () => {
const a = seed('s-dead-1');
const b = seed('s-healthy-2');
__setReloginDeps({ windsurfLogin: async () => { throw new Error('no credential'); } });
const seen = [];
__setConnectDeps({
streamChatCompletion: async (params, send) => {
seen.push(params.token);
if (params.token === a.apiKey) throw unauthorized(); // dead before first chunk
send({ id: 's', object: 'chat.completion.chunk', choices: [{ index: 0, delta: { content: 'OK' }, finish_reason: null }] });
send({ id: 's', object: 'chat.completion.chunk', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 200);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
assert.ok(frames.some(f => f !== '[DONE]' && f.choices?.[0]?.delta?.content === 'OK'), 'streamed content from healthy account');
assert.ok(seen.includes(a.apiKey) && seen.includes(b.apiKey), 'failed over after dead account');
assert.equal(frames.at(-1), '[DONE]');
});
it('does NOT fail over once a chunk has already been emitted', async () => {
const a = seed('s-mid-1');
seed('s-mid-2');
__setReloginDeps({ windsurfLogin: async () => { throw new Error('no credential'); } });
const seen = [];
__setConnectDeps({
streamChatCompletion: async (params, send) => {
seen.push(params.token);
// Emit one chunk, THEN fail. A retry would duplicate content, so the
// loop must surface an error frame instead of hopping accounts.
send({ id: 's', object: 'chat.completion.chunk', choices: [{ index: 0, delta: { content: 'partial' }, finish_reason: null }] });
throw unauthorized();
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
assert.equal(seen.length, 1, 'no failover after a byte is on the wire');
assert.ok(frames.some(f => f !== '[DONE]' && f.choices?.[0]?.delta?.content === 'partial'), 'emitted the partial chunk');
assert.ok(frames.some(f => f !== '[DONE]' && f.error), 'surfaced an error frame');
});
// A pre-content error must not be closed with [DONE]: the OpenAI stream
// contract treats [DONE] as a completed turn, which turns an upstream failure
// into a clean empty answer. Content already on the wire keeps the historical
// clean-close behavior (the previous test).
it('a pre-content upstream error ends without [DONE]', async () => {
seed('s-pre-err');
__setConnectDeps({
streamChatCompletion: async () => {
throw Object.assign(new Error('blocked by upstream'), { code: 'CONTENT_BLOCKED' });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
assert.ok(frames.some(f => f !== '[DONE]' && f.error), 'surfaced an error frame');
assert.equal(frames.includes('[DONE]'), false, 'no [DONE] after a pre-content error');
});
// A pre-content 429 must keep the rate-limit class on the wire; OpenAI SDKs
// back off on rate_limit_error, not on api_error.
it('a pre-content 429 keeps rate_limit_error on the stream', async () => {
seed('s-429');
__setConnectDeps({
streamChatCompletion: async () => {
throw Object.assign(new Error('rate limited'), { code: 'RATE_LIMITED' });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
const err = frames.find(f => f !== '[DONE]' && f.error);
assert.equal(err?.error?.type, 'rate_limit_error');
assert.equal(frames.includes('[DONE]'), false);
});
// R1 (stream): a pre-emit RATE_LIMITED on the first account fails over to a
// healthy one instead of erroring the stream — replay is safe while !emitted.
it('R1 stream: fails over on a pre-emit RATE_LIMITED, then streams from the healthy account', async () => {
const a = seed('s-rl-1');
seed('s-rl-2');
const seen = [];
__setConnectDeps({
streamChatCompletion: async (params, send) => {
seen.push(params.token);
if (params.token === a.apiKey) throw Object.assign(new Error('rate limited'), { code: 'RATE_LIMITED' });
send({ id: 's', object: 'chat.completion.chunk', choices: [{ index: 0, delta: { content: 'OK' }, finish_reason: null }] });
send({ id: 's', object: 'chat.completion.chunk', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
assert.equal(seen.length, 2, 'RATE_LIMITED on account 1 triggered one stream failover hop');
assert.ok(frames.some(f => f !== '[DONE]' && f.choices?.[0]?.delta?.content === 'OK'), 'streamed from the healthy account');
assert.ok(!frames.some(f => f !== '[DONE]' && f.error), 'no error frame — the request was recovered');
});
// R1 guard (stream): once a byte is on the wire, a RATE_LIMITED mid-stream must
// NOT hop (a replay would duplicate content) — it surfaces an error frame.
it('R1 stream: does NOT fail over on RATE_LIMITED after a chunk was emitted', async () => {
seed('s-rlmid-1');
seed('s-rlmid-2');
const seen = [];
__setConnectDeps({
streamChatCompletion: async (params, send) => {
seen.push(params.token);
send({ id: 's', object: 'chat.completion.chunk', choices: [{ index: 0, delta: { content: 'partial' }, finish_reason: null }] });
throw Object.assign(new Error('rate limited'), { code: 'RATE_LIMITED' });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
assert.equal(seen.length, 1, 'no failover after a byte is on the wire, even for RATE_LIMITED');
assert.ok(frames.some(f => f !== '[DONE]' && f.choices?.[0]?.delta?.content === 'partial'), 'emitted the partial chunk');
assert.ok(frames.some(f => f !== '[DONE]' && f.error), 'surfaced an error frame');
});
it('stream free→paid: fresh token still UNAUTHORIZED ⇒ MODEL_BLOCKED error frame, no storm (#42)', async () => {
// Streaming twin of the non-stream entitlement-wall test. Nothing is emitted
// (the wall hits before any byte), so re-login fires once as a disambiguation
// probe; the fresh token also 401s ⇒ MODEL_BLOCKED error frame, NOT a failover
// hop to the second account.
const a = seed('s-paidwall-1');
const b = seed('s-paidwall-2');
process.env.DEVIN_CONNECT_AUTO_RELOGIN = '1';
__setReloginDeps({
isCredStoreEnabled: () => true,
getCredential: () => 'stored-password',
windsurfLogin: async () => ({ apiKey: `devin-session-token$fresh-${Math.random().toString(36).slice(2)}`, name: 's-paidwall-1' }),
});
const seen = [];
__setConnectDeps({
streamChatCompletion: async (params) => { seen.push(params.token); throw unauthorized(); },
});
const result = await handleChatCompletions(
{ model: 'claude-opus-4.8', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
const errFrame = frames.find(f => f !== '[DONE]' && f.error);
assert.ok(errFrame, 'surfaced an error frame');
assert.equal(errFrame.error.code, 'MODEL_BLOCKED', 'entitlement wall, not dead-token');
assert.ok(!seen.includes(b.apiKey), 'did not fail over to a second account');
});
it('does NOT replay a pre-emit transient 5xx in the handler (the adapter owns that retry, P1 #34)', async () => {
seed('s-transient-1');
let calls = 0;
__setConnectDeps({
streamChatCompletion: async () => {
calls++;
throw transient(); // 503 before any byte
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
// Q5 ownership: devin-connect-openai.js performs the single bounded
// pre-emit transport retry; the handler must not add a second one.
assert.equal(calls, 1, 'one adapter call — the adapter owns the bounded retry');
assert.ok(frames.some(f => f !== '[DONE]' && f.error), 'the transient error still reaches the client');
});
it('CAPACITY (high-demand) never re-logs in or fails over in the handler (P0 #56/#57)', async () => {
// The regression that motivated this: a momentary "high demand" on a free
// token was classified UNAUTHORIZED → auto re-login → still busy → mislabeled
// MODEL_BLOCKED → free model cooled down forever. The single in-place replay
// is now owned by the adapter (retry-ownership.test.js); this test pins the
// handler half: a capacity blip must NEVER re-login or hop accounts.
const a = seed('s-capacity-1');
const b = seed('s-capacity-2');
process.env.DEVIN_CONNECT_AUTO_RELOGIN = '1';
let reloginCalls = 0;
__setReloginDeps({
isCredStoreEnabled: () => true,
getCredential: () => 'stored-password',
windsurfLogin: async () => { reloginCalls++; return { apiKey: 'devin-session-token$should-never-mint', name: 's-capacity-1' }; },
});
let calls = 0;
const seen = [];
__setConnectDeps({
streamChatCompletion: async (params) => {
calls++;
seen.push(params.token);
throw capacity(); // model busy, before any byte
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
assert.equal(calls, 1, 'one handler attempt — the adapter owns the in-place replay');
assert.equal(reloginCalls, 0, 'CAPACITY must NEVER trigger re-login');
assert.ok(!seen.includes(b.apiKey), 'did not fail over to the second account');
assert.ok(frames.some(f => f !== '[DONE]' && f.error), 'surfaced the capacity error');
});
it('persistent CAPACITY surfaces a clean 503-class error, NOT MODEL_BLOCKED or a re-login storm (P0 #57)', async () => {
// If the model stays busy through the one in-place replay, we surface the
// capacity error as-is. It must NOT escalate to MODEL_BLOCKED (permanent
// cooldown) and must NOT have stormed re-login across the pool.
seed('s-capacity-persist-1');
seed('s-capacity-persist-2');
process.env.DEVIN_CONNECT_AUTO_RELOGIN = '1';
let reloginCalls = 0;
__setReloginDeps({
isCredStoreEnabled: () => true,
getCredential: () => 'stored-password',
windsurfLogin: async () => { reloginCalls++; return { apiKey: 'devin-session-token$nope', name: 's-capacity-persist-1' }; },
});
let calls = 0;
__setConnectDeps({
streamChatCompletion: async () => { calls++; throw capacity(); }, // busy forever
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
const errFrame = frames.find(f => f !== '[DONE]' && f.error);
assert.equal(calls, 1, 'one handler attempt — the adapter owns the in-place replay (no pool storm)');
assert.equal(reloginCalls, 0, 'CAPACITY must NEVER trigger re-login');
assert.ok(errFrame, 'surfaced an error frame');
assert.notEqual(errFrame.error.code, 'MODEL_BLOCKED', 'capacity must NOT escalate to a permanent entitlement wall');
assert.equal(errFrame.error.code, 'CAPACITY', 'surfaced the capacity error as-is');
});
it('does NOT replay a transient error once a byte is already emitted', async () => {
seed('s-transient-mid-1');
let calls = 0;
__setConnectDeps({
streamChatCompletion: async (params, send) => {
calls++;
send({ id: 's', object: 'chat.completion.chunk', choices: [{ index: 0, delta: { content: 'partial' }, finish_reason: null }] });
throw transient(); // 503 AFTER a byte → replay would duplicate, so it must not
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
const res = fakeRes();
await result.handler(res);
const frames = parseFrames(res.body);
assert.equal(calls, 1, 'no replay after a byte is on the wire');
assert.ok(frames.some(f => f !== '[DONE]' && f.error), 'surfaced an error frame instead');
});
});
describe('DEVIN_CONNECT quota vs tier wall (P1 #33)', () => {
it('QUOTA_EXHAUSTED cools the account down AND surfaces 402 insufficient_quota', async () => {
const a = seed('quota-1');
process.env.DEVIN_CONNECT_FAILOVER_MAX = '0'; // isolate this account
__setConnectDeps({
toChatCompletion: async () => {
throw Object.assign(new Error('insufficient credits remaining'), { code: 'QUOTA_EXHAUSTED' });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 402);
assert.equal(result.body.error.type, 'insufficient_quota');
// R6: the dry account is now cooled on the QUOTA dimension (quotaResetAt),
// the same self-healing dimension a proactive credits snapshot uses — not the
// transient rateLimitedUntil. Availability reports it distinctly as
// 'quota_exhausted' so selection skips it (the distinguishing penalty vs a
// tier wall, which stays fully available).
const avail = getAccountAvailability(a.apiKey, 'swe-1-6-slow');
assert.equal(avail.reason, 'quota_exhausted', 'dry account is cooled on the quota dimension');
assert.ok(avail.retryAfterMs > 0, 'cooldown carries a retry hint');
});
it('the LIVE quota-exhausted body reaches the dry-well path, not the internal-error breaker', async () => {
// The whole point of the classifier fix, exercised end to end: take the exact
// upstream body through the REAL classifier (what devin-connect.js does when it
// sees a non-200), throw the classified shape at the handler, and assert the
// account is cooled on the quota dimension rather than charged an internal error.
//
// Before the fix the classified code was `failed_precondition`, which is in
// TRANSPORT_FAULT_CODES → reportInternalError, and the request surfaced 502.
const a = seed('quota-live');
process.env.DEVIN_CONNECT_FAILOVER_MAX = '0';
const live = 'Your daily usage quota has been exhausted. Visit '
+ 'https://app.devin.ai/settings/usage to review your plan and usage. (trace ID: 6f1c9a2e)';
__setConnectDeps({
toChatCompletion: async () => {
const classified = classifyUpstreamError(live, 'failed_precondition', 400);
throw Object.assign(new Error(live), classified);
},
});
const result = await handleChatCompletions(
{ model: 'swe-2-medium', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 402);
assert.equal(result.body.error.type, 'insufficient_quota');
const internal = getAccountInternal(a.id);
assert.equal(internal.internalErrorStreak || 0, 0,
'a quota-dry account is healthy; charging the internal-error breaker quarantines it '
+ 'after 2 hits and takes the only working account offline');
assert.ok(internal.quotaResetAt > Date.now(), 'cooled on the self-healing quota dimension');
assert.equal(getAccountAvailability(a.apiKey, 'swe-2-medium').reason, 'quota_exhausted');
});
it('MODEL_BLOCKED (tier wall) returns 402 but does NOT cool the account down', async () => {
const a = seed('tier-1');
process.env.DEVIN_CONNECT_FAILOVER_MAX = '0';
__setConnectDeps({
toChatCompletion: async () => {
throw Object.assign(new Error('please /upgrade to access this model'), { code: 'MODEL_BLOCKED' });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 402);
assert.equal(result.body.error.type, 'model_blocked');
// No rate-limit cooldown was added — a healthy account hitting a paid wall
// must not be demoted. (Fresh token accounts read as model_not_available by
// default; the point is the reason is NOT a rate_limit penalty.)
const avail = getAccountAvailability(a.apiKey, 'swe-1-6-slow');
assert.notEqual(avail.reason, 'rate_limited', 'tier wall must not cool the account down');
});
it('persistent CAPACITY applies a SHORT model-scoped soft cooldown, account stays healthy for other models (P0 #56/#57)', async () => {
const a = seed('cap-nonstream-1');
process.env.DEVIN_CONNECT_FAILOVER_MAX = '0'; // isolate this account
__setConnectDeps({
toChatCompletion: async () => {
throw Object.assign(new Error("We're currently facing high demand for this model. Please try again later."), { code: 'CAPACITY', status: 401 });
},
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 503, 'capacity surfaces as 503, not 401/402');
assert.equal(result.body.error.type, 'capacity_error');
// The model that was busy carries a short cooldown so the pool prefers
// another account for THIS model...
const availBusy = getAccountAvailability(a.apiKey, 'swe-1-6-slow');
assert.equal(availBusy.reason, 'model_rate_limited', 'busy model is softly cooled down');
assert.ok(availBusy.retryAfterMs > 0 && availBusy.retryAfterMs <= 60 * 1000, 'short (≤60s) cooldown, auto-recovering');
// ...but the account is NOT cooled down account-wide — every OTHER model is
// still fully serviceable (the cooldown is model-scoped, not a fault penalty).
const availOther = getAccountAvailability(a.apiKey, 'gemini-2.5-flash');
assert.notEqual(availOther.reason, 'rate_limited', 'account stays healthy for other models');
assert.notEqual(availOther.reason, 'model_rate_limited', 'other model not cooled down');
});
});
describe('DEVIN_CONNECT pool exhaustion (P0-2)', () => {
it('returns a clean 429 with retry_after when the whole pool is rate-limited', async () => {
const a = seed('exhaust-1');
const b = seed('exhaust-2');
// Whole pool rate-limited → acquire would yield nothing. Must NOT silently
// fall back to the un-accounted env token; must return a fast 429.
markRateLimited(a.apiKey, 5 * 60 * 1000);
markRateLimited(b.apiKey, 5 * 60 * 1000);
let upstreamHits = 0;
__setConnectDeps({ toChatCompletion: async () => { upstreamHits++; return { status: 200, body: {} }; } });
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(result.status, 429);
assert.equal(result.body.error.type, 'rate_limit_exceeded');
assert.ok(result.body.error.retry_after_ms > 0, 'carries a retry_after hint');
assert.ok(result.headers['Retry-After'], 'sets the Retry-After header');
assert.equal(upstreamHits, 0, 'never touched upstream on an exhausted pool');
});
it('still falls back to the env token when the pool is genuinely EMPTY (single-token deploy)', async () => {
// No accounts seeded. getAccountCount().total === 0 → exhaustion guard is
// skipped and the connect client uses the env token, preserving the
// single-token deploy story.
const prevToken = process.env.DEVIN_CONNECT_TOKEN;
process.env.DEVIN_CONNECT_TOKEN = 'devin-session-token$env-fallback';
try {
__setConnectDeps({
toChatCompletion: async () => (
{ status: 200, body: { id: 'x', choices: [{ message: { role: 'assistant', content: 'ENV_OK' } }] } }
),
});
const result = await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
// Empty pool must NOT 429 — it serves via the env-token fallback.
assert.equal(result.status, 200);
assert.equal(result.body.choices[0].message.content, 'ENV_OK');
} finally {
if (prevToken === undefined) delete process.env.DEVIN_CONNECT_TOKEN;
else process.env.DEVIN_CONNECT_TOKEN = prevToken;
}
});
});
describe('DEVIN_CONNECT re-login storm guard (P0-1)', () => {
it('a burst of concurrent dead-token requests triggers exactly one windsurfLogin', async () => {
// The cutover-day scenario: a batch of session tokens dies at once and many
// in-flight requests hit the same dead account. The request path must NOT
// force-bypass the cooldown — concurrent callers coalesce on auth.js's
// inflight promise, so the account re-logs in ONCE, not once-per-request.
const a = seed('storm-1');
process.env.DEVIN_CONNECT_AUTO_RELOGIN = '1';
process.env.DEVIN_CONNECT_FAILOVER_MAX = '0'; // single account, no hop
let loginCalls = 0;
__setReloginDeps({
isCredStoreEnabled: () => true,
getCredential: () => 'stored-password',
// Slow + failing: stays inflight through the burst, never recovers, so
// every request reaches the re-login decision point under contention.
windsurfLogin: async () => { loginCalls++; await new Promise(r => setTimeout(r, 25)); throw new Error('auth1 down'); },
});
__setConnectDeps({ toChatCompletion: async () => { throw unauthorized(); } });
const burst = await Promise.all(
Array.from({ length: 6 }, () => handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
)),
);
assert.ok(burst.every(r => r.status === 401), 'all requests get a clean 401 when recovery fails');
assert.equal(loginCalls, 1, 'one Auth1 login for the whole burst — no storm');
});
});
describe('DEVIN_CONNECT sampling passthrough (#48)', () => {
it('forwards temperature/top_p/top_k/max_tokens from the request into connect params', async () => {
seed('sampling-1');
let captured = null;
__setConnectDeps({
toChatCompletion: async (params) => {
captured = params.completion;
return { status: 200, body: { id: 'x', choices: [{ message: { role: 'assistant', content: 'OK' } }] } };
},
});
const result = await handleChatCompletions(
{
model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }],
temperature: 0.3, top_p: 0.8, top_k: 50, max_tokens: 200,
},
{ callerKey: '' },
);
assert.equal(result.status, 200);
assert.deepEqual(captured, { temperature: 0.3, topP: 0.8, topK: 50, maxTokens: 200 });
});
it('omits the completion override entirely when the caller sends no sampling params', async () => {
seed('sampling-2');
let hadCompletion = true;
__setConnectDeps({
toChatCompletion: async (params) => {
hadCompletion = 'completion' in params;
return { status: 200, body: { id: 'x', choices: [{ message: { role: 'assistant', content: 'OK' } }] } };
},
});
await handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: '' },
);
assert.equal(hadCompletion, false, 'no completion key when caller specified no sampling controls');
});
});
// O3: newer OpenAI SDKs and the o1/o3/gpt-5 reasoning families send
// `max_completion_tokens` instead of `max_tokens`. It must be honored as the
// output cap, and take precedence when both are present.
describe('O3: max_completion_tokens output cap', () => {
function captureCompletion(label, body) {
seed(label);
let captured = 'unset';
__setConnectDeps({
toChatCompletion: async (params) => {
captured = params.completion;
return { status: 200, body: { id: 'x', choices: [{ message: { role: 'assistant', content: 'OK' } }] } };
},
});
return handleChatCompletions(
{ model: 'swe-1-6-slow', stream: false, messages: [{ role: 'user', content: 'hi' }], ...body },
{ callerKey: '' },
).then((result) => ({ result, get captured() { return captured; } }));
}
it('forwards max_completion_tokens as the connect maxTokens cap', async () => {
const { result, captured } = await captureCompletion('o3-mct', { max_completion_tokens: 321 });
assert.equal(result.status, 200);
assert.deepEqual(captured, { maxTokens: 321 });
});
it('prefers max_completion_tokens over max_tokens when both are sent', async () => {
// OpenAI precedence: the modern field wins. A client migrating mid-flight that
// sends both must get the o1-style cap, not the legacy one.
const { captured } = await captureCompletion('o3-both', { max_completion_tokens: 500, max_tokens: 100 });
assert.deepEqual(captured, { maxTokens: 500 });
});
it('still honors legacy max_tokens when max_completion_tokens is absent', async () => {
const { captured } = await captureCompletion('o3-legacy', { max_tokens: 77 });
assert.deepEqual(captured, { maxTokens: 77 });
});
it('ignores a non-finite max_completion_tokens and falls back to max_tokens', async () => {
// A null/garbage modern field must not shadow a valid legacy cap.
const { captured } = await captureCompletion('o3-nan', { max_completion_tokens: null, max_tokens: 42 });
assert.deepEqual(captured, { maxTokens: 42 });
});
});