test/devin-acp-integration.test.js
Rama master · réplica de lectura del repositorio original.
import { afterEach, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { handleChatCompletions } from '../src/handlers/chat.js';
import { __resetDevinAcpProbeCache } from '../src/devin-acp.js';
import {
getModelAccessConfig,
setModelAccessList,
setModelAccessMode,
} from '../src/dashboard/model-access.js';
// S3 coverage: PRODUCT-LEVEL integration of the ACP escape hatch.
//
// The existing suites cover the two ends separately:
// - special-agent-routing.test.js mocks `runDevinAcp` at the HTTP boundary.
// - devin-acp-edge.test.js drives the REAL runner against a fake subprocess.
// Neither joins both ends. These tests run the FULL real path —
// handleChatCompletions -> handleSpecialAgentChatCompletion -> real
// runDevinAcp -> real runDevinAcpProcess -> fake `devin acp` subprocess
// — so account-pool credential injection, response shaping, SSE framing and
// HTTP error mapping are all exercised together with no mocked seam in the
// middle. The fake stdio server stands in for the real Devin CLI binary: no
// real account, no real binary, no network.
const ENV_KEYS = [
'WINDSURFAPI_SPECIAL_AGENT_BACKEND',
'DEVIN_CLI_ENABLED',
'DEVIN_CLI_MODE',
'DEVIN_CLI_PATH',
'DEVIN_CLI_ACP_ARGS_JSON',
'DEVIN_CLI_USE_ACCOUNT_POOL',
'DEVIN_TIMEOUT_MS',
'DEVIN_OUTPUT_LIMIT_BYTES',
'DEVIN_CLI_WORKDIR',
'DEVIN_ONLY',
'DEVIN_ACP_PROBE',
];
const originalEnv = Object.fromEntries(ENV_KEYS.map(k => [k, process.env[k]]));
const originalModelAccess = getModelAccessConfig();
const tmpDirs = [];
beforeEach(() => {
// The proactive availability probe (default ON) caches its verdict keyed on
// the resolved CLI path. Reset between tests so a prior test's path verdict
// never leaks into the next. installAcpBackend points at process.execPath
// (node --version exits 0 → available), so the real path still runs.
__resetDevinAcpProbeCache();
});
afterEach(() => {
for (const k of ENV_KEYS) {
if (originalEnv[k] === undefined) delete process.env[k];
else process.env[k] = originalEnv[k];
}
setModelAccessMode(originalModelAccess.mode || 'all');
setModelAccessList(originalModelAccess.list || []);
while (tmpDirs.length) {
try { rmSync(tmpDirs.pop(), { recursive: true, force: true }); } catch { /* ignore */ }
}
});
// Install a fake `devin acp` stdio server, enable the acp backend, and point
// the real runner at it via env. `timeoutMs` drives DEVIN_TIMEOUT_MS (floor
// 1000ms in runTimeoutMs()).
function installAcpBackend(source, { timeoutMs = 5000 } = {}) {
const dir = mkdtempSync(join(tmpdir(), 'windsurfapi-acp-int-'));
tmpDirs.push(dir);
const script = join(dir, 'fake-acp.mjs');
writeFileSync(script, source, 'utf8');
process.env.WINDSURFAPI_SPECIAL_AGENT_BACKEND = 'devin-cli';
process.env.DEVIN_CLI_MODE = 'acp';
process.env.DEVIN_CLI_PATH = process.execPath;
process.env.DEVIN_CLI_ACP_ARGS_JSON = JSON.stringify([script]);
process.env.DEVIN_TIMEOUT_MS = String(timeoutMs);
return script;
}
function poolContext(account, hooks = {}) {
return {
specialAgent: {
checkoutAccount: hooks.checkoutAccount || (() => account),
releaseAccount: hooks.releaseAccount || (() => {}),
// Intentionally NOT overriding runDevinAcp: the REAL runner must run.
},
};
}
// A fake that completes the full handshake and echoes back, as its visible
// reply, the prompt text AND the api_key/server it was authenticated with.
// That lets a test prove end-to-end that the account-pool credential reached
// the ACP authenticate frame and that the model hint reached session/prompt.
const ECHO_FAKE = `
import readline from 'node:readline';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
function update(u) { send({ jsonrpc: '2.0', method: 'session/update', params: { sessionId: 'session-1', update: u } }); }
let seenKey = '';
let seenServer = '';
rl.on('line', line => {
const msg = JSON.parse(line);
if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
if (msg.method === 'authenticate') {
seenKey = msg.params?._meta?.api_key || '';
seenServer = msg.params?._meta?.api_server_url || '';
send({ jsonrpc: '2.0', id: msg.id, result: {} });
return;
}
if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; }
if (msg.method === 'session/prompt') {
const promptText = (msg.params?.prompt || []).map(p => p.text || '').join('');
update({ sessionUpdate: 'agent_thought_chunk', content: { text: 'thinking...' } });
update({ sessionUpdate: 'agent_message_chunk', content: { text: 'KEY=' + seenKey + '|SERVER=' + seenServer + '|PROMPT<' + promptText + '>' } });
send({ jsonrpc: '2.0', id: msg.id, result: { stopReason: 'end_turn', usage: { totalTokens: 7 } } });
return;
}
});
`;
// A minimal fake that just replies with a fixed message after the handshake.
// Used where the test only cares about routing/shape, not credential echo.
function fixedReplyFake(reply) {
return `
import readline from 'node:readline';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
function update(u) { send({ jsonrpc: '2.0', method: 'session/update', params: { sessionId: 'session-1', update: u } }); }
rl.on('line', line => {
const msg = JSON.parse(line);
if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; }
if (msg.method === 'session/prompt') {
update({ sessionUpdate: 'agent_message_chunk', content: { text: ${JSON.stringify(reply)} } });
send({ jsonrpc: '2.0', id: msg.id, result: { stopReason: 'end_turn' } });
return;
}
});
`;
}
describe('ACP integration — full HTTP -> ACP -> response path', () => {
it('runs swe-1.6 end-to-end and injects the account-pool credential into the ACP authenticate frame', async () => {
installAcpBackend(ECHO_FAKE);
let released = '';
const account = {
id: 'acct-pool-1',
apiKey: 'pool-key-abc123',
apiServerUrl: 'https://server.self-serve.windsurf.com',
};
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'integration ping' }] },
poolContext(account, { releaseAccount: (k) => { released = k; } }),
);
assert.equal(result.status, 200);
const content = result.body.choices[0].message.content;
// Proves the pool credential flowed all the way to ACP authenticate.
assert.match(content, /KEY=pool-key-abc123/);
assert.match(content, /SERVER=https:\/\/server\.self-serve\.windsurf\.com/);
// Proves the user prompt + model hint reached session/prompt.
assert.match(content, /PROMPT<[\s\S]*Model requested by caller: swe-1\.6[\s\S]*integration ping[\s\S]*>/);
// Thought chunk must NOT leak into the visible reply.
assert.doesNotMatch(content, /thinking\.\.\./);
assert.equal(result.body.model, 'swe-1.6');
assert.ok(result.body.usage.completion_tokens > 0);
// The borrowed account must be returned to the pool by its apiKey.
assert.equal(released, 'pool-key-abc123');
});
it('routes every special_agent model id through the real ACP runner', async () => {
// swe-1.6 is covered above; verify the rest of the catalog reaches ACP too.
for (const model of ['swe-1.6-fast', 'adaptive', 'arena-fast', 'arena-smart']) {
installAcpBackend(fixedReplyFake(`REPLY_FOR_${model}`));
const result = await handleChatCompletions(
{ model, messages: [{ role: 'user', content: `hello ${model}` }] },
poolContext({ id: 'a', apiKey: 'k', apiServerUrl: '' }),
);
assert.equal(result.status, 200, `${model} should return 200`);
assert.equal(result.body.choices[0].message.content, `REPLY_FOR_${model}`);
assert.equal(result.body.model, model);
}
});
it('omits api_server_url from the authenticate frame when the pool account has none', async () => {
installAcpBackend(ECHO_FAKE);
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'no server url' }] },
poolContext({ id: 'a', apiKey: 'key-only', apiServerUrl: '' }),
);
assert.equal(result.status, 200);
const content = result.body.choices[0].message.content;
assert.match(content, /KEY=key-only/);
assert.match(content, /SERVER=\|/); // empty server segment
});
});
describe('ACP integration — streaming (SSE)', () => {
// NOTE on streaming semantics: the special-agent path does NOT stream token
// deltas from ACP. It runs the ACP request to completion, then re-frames the
// final text as a buffered SSE (streamFromText in special-agent.js). So an
// incremental token stream from special_agent is NOT supported today; the
// SSE contract here is "one buffered content delta + finish + [DONE]". This
// test pins that real behavior end-to-end so a future real-streaming change
// is a conscious, test-visible decision.
it('serves a buffered SSE stream from the real ACP runner for stream:true requests', async () => {
installAcpBackend(fixedReplyFake('STREAMED_ACP_BODY'));
const result = await handleChatCompletions(
{ model: 'swe-1.6', stream: true, messages: [{ role: 'user', content: 'stream please' }] },
poolContext({ id: 'a', apiKey: 'k', apiServerUrl: '' }),
);
assert.equal(result.status, 200);
assert.equal(result.stream, true);
assert.match(result.headers['Content-Type'], /text\/event-stream/);
const writes = [];
const res = {
writableEnded: false,
write(chunk) { writes.push(String(chunk)); },
end() { this.writableEnded = true; },
};
await result.handler(res);
const joined = writes.join('');
assert.match(joined, /STREAMED_ACP_BODY/);
assert.match(joined, /"object":"chat\.completion\.chunk"/);
assert.match(joined, /finish_reason":"stop"/);
assert.match(joined, /data: \[DONE\]/);
assert.equal(res.writableEnded, true);
});
});
describe('ACP integration — error propagation to HTTP status', () => {
it('maps a transient "overloaded" ACP error to 429 capacity_error (transient-first, account NOT burned)', async () => {
// AC2 P0: the upstream wraps capacity faults in an RPC error. The ACP path
// must classify them transient-first so reportRunFailure cools the account
// down (markRateLimited) instead of feeding the disable-streak. Surfaced as
// 429 capacity_error end-to-end through the HTTP boundary.
installAcpBackend(`
import readline from 'node:readline';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
rl.on('line', line => {
const msg = JSON.parse(line);
if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; }
if (msg.method === 'session/prompt') { send({ jsonrpc: '2.0', id: msg.id, error: { code: -32000, message: 'model overloaded' } }); return; }
});
`);
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'will fail' }] },
poolContext({ id: 'a', apiKey: 'k', apiServerUrl: '' }),
);
assert.equal(result.status, 429);
assert.equal(result.body.error.type, 'capacity_error');
assert.equal(result.body.error.backend, 'devin-cli');
});
it('maps an unrecognised ACP RPC error to 502 backend_error (legacy fallback unchanged)', async () => {
installAcpBackend(`
import readline from 'node:readline';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
rl.on('line', line => {
const msg = JSON.parse(line);
if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; }
if (msg.method === 'session/prompt') { send({ jsonrpc: '2.0', id: msg.id, error: { code: -32000, message: 'malformed prompt blob' } }); return; }
});
`);
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'will fail' }] },
poolContext({ id: 'a', apiKey: 'k', apiServerUrl: '' }),
);
assert.equal(result.status, 502);
assert.equal(result.body.error.type, 'backend_error');
assert.equal(result.body.error.backend, 'devin-cli');
});
it('maps a missing sessionId from session/new to 502 backend_error', async () => {
installAcpBackend(`
import readline from 'node:readline';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
rl.on('line', line => {
const msg = JSON.parse(line);
if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
});
`);
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'no session' }] },
poolContext({ id: 'a', apiKey: 'k', apiServerUrl: '' }),
);
assert.equal(result.status, 502);
assert.equal(result.body.error.type, 'backend_error');
});
it('maps a stalled ACP prompt to 504 backend_timeout', async () => {
installAcpBackend(`
import readline from 'node:readline';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
rl.on('line', line => {
const msg = JSON.parse(line);
if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; }
// session/prompt: intentionally never answered.
});
`, { timeoutMs: 1000 });
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'stall' }] },
poolContext({ id: 'a', apiKey: 'k', apiServerUrl: '' }),
);
assert.equal(result.status, 504);
assert.equal(result.body.error.type, 'backend_timeout');
});
it('returns 503 pool_exhausted when the account pool has no account (never spawns ACP)', async () => {
installAcpBackend(ECHO_FAKE);
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'no account' }] },
poolContext(null, { checkoutAccount: () => null }),
);
assert.equal(result.status, 503);
assert.equal(result.body.error.type, 'pool_exhausted');
});
it('maps a missing Devin CLI binary (ENOENT) to 503 backend_unavailable', async () => {
process.env.WINDSURFAPI_SPECIAL_AGENT_BACKEND = 'devin-cli';
process.env.DEVIN_CLI_MODE = 'acp';
process.env.DEVIN_CLI_PATH = join(tmpdir(), 'no-such-devin-binary-int-987');
process.env.DEVIN_CLI_ACP_ARGS_JSON = JSON.stringify(['acp']);
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'no binary' }] },
poolContext({ id: 'a', apiKey: 'k', apiServerUrl: '' }),
);
assert.equal(result.status, 503);
assert.equal(result.body.error.type, 'backend_unavailable');
});
// GAP-ACP-01: a clean exit (code 0) WHILE a request is in flight must settle
// the pending promise instead of silently dropping it. Before the fix, the
// close handler called cleanup() on code 0, clearing the pending map without
// rejecting — the awaiting caller hung until the (already cleared) timeout
// that never fires, permanently leaking the concurrency slot. The proof:
// the call RETURNS (with a 502, well under the timeout) and the account is
// released, rather than hanging until DEVIN_TIMEOUT_MS.
it('GAP-ACP-01: a clean process exit mid-prompt settles the request and releases the slot', async () => {
// Fake completes the handshake, then on session/prompt exits 0 WITHOUT
// ever sending the result frame.
installAcpBackend(`
import readline from 'node:readline';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
rl.on('line', line => {
const msg = JSON.parse(line);
if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; }
if (msg.method === 'session/prompt') { process.exit(0); }
});
`, { timeoutMs: 10_000 });
let released = false;
const started = Date.now();
const result = await handleChatCompletions(
{ model: 'swe-1.6', messages: [{ role: 'user', content: 'clean exit' }] },
poolContext({ id: 'a', apiKey: 'k', apiServerUrl: '' }, { releaseAccount: () => { released = true; } }),
);
const elapsed = Date.now() - started;
assert.equal(result.status, 502, 'clean exit before result maps to 502');
assert.equal(result.body.error.type, 'backend_error');
assert.ok(released, 'account slot was released (not leaked)');
assert.ok(elapsed < 9_000, `returned in ${elapsed}ms, well under the 10s timeout (did not hang)`);
});
});
// DEVIN_ONLY end-to-end: Cascade is retired, so a model that would normally
// take the Cascade Connect-RPC flow (e.g. claude-sonnet-4.6) must instead run
// through the real Devin ACP runner — with no WINDSURFAPI_SPECIAL_AGENT_BACKEND
// set, because DEVIN_ONLY alone enables the whole path. This proves the single
// kill-switch actually re-routes normal models, not just the special-agent
// catalog. (It does NOT prove Devin serves claude as claude — the model name
// only reaches the prompt hint; true model selection is gated on a live probe.)
describe('ACP integration — DEVIN_ONLY routes a normal (Cascade) model into Devin', () => {
it('forces claude-sonnet-4.6 through the real ACP runner under DEVIN_ONLY=1', async () => {
process.env.DEVIN_ONLY = '1';
// Deliberately do NOT set WINDSURFAPI_SPECIAL_AGENT_BACKEND — DEVIN_ONLY
// must enable the backend on its own.
delete process.env.WINDSURFAPI_SPECIAL_AGENT_BACKEND;
delete process.env.DEVIN_CLI_ENABLED;
installAcpBackend(ECHO_FAKE);
const result = await handleChatCompletions(
{ model: 'claude-sonnet-4.6', messages: [{ role: 'user', content: 'devin-only ping' }] },
poolContext({ id: 'a', apiKey: 'pool-key-claude', apiServerUrl: 'https://srv.example' }),
);
assert.equal(result.status, 200, 'claude routed into Devin returns 200');
const content = result.body.choices[0].message.content;
// The ECHO fake proves the request reached the ACP authenticate + prompt
// frames — i.e. it went through Devin, not Cascade.
assert.match(content, /KEY=pool-key-claude/);
assert.match(content, /PROMPT<[\s\S]*Model requested by caller: claude-sonnet-4\.6[\s\S]*devin-only ping[\s\S]*>/);
assert.equal(result.body.model, 'claude-sonnet-4.6', 'response echoes the requested model id');
});
});