test/devin-acp-transient.test.js

Rama master · réplica de lectura del repositorio original.

import { afterEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, writeFileSync, rmSync, readFileSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { runDevinAcpProcess, classifyAcpError } from '../src/devin-acp.js';

// AC2 P0 + P1: prove the ACP path is transient-first (never burns a healthy
// account on a retryable upstream hiccup) and that abort sends a graceful
// `session/cancel` notification before SIGTERM. All against a FAKE stdio child
// — no real Devin CLI, no account, no network.

const ENV_KEYS = [
  'DEVIN_CLI_PATH',
  'DEVIN_CLI_ACP_ARGS_JSON',
  'DEVIN_TIMEOUT_MS',
  'DEVIN_OUTPUT_LIMIT_BYTES',
  'DEVIN_CLI_WORKDIR',
];
const originalEnv = Object.fromEntries(ENV_KEYS.map(k => [k, process.env[k]]));
const tmpDirs = [];

afterEach(() => {
  for (const k of ENV_KEYS) {
    if (originalEnv[k] === undefined) delete process.env[k];
    else process.env[k] = originalEnv[k];
  }
  while (tmpDirs.length) {
    try { rmSync(tmpDirs.pop(), { recursive: true, force: true }); } catch { /* ignore */ }
  }
});

function installFakeAcp(source, { timeoutMs = 5000 } = {}) {
  const dir = mkdtempSync(join(tmpdir(), 'windsurfapi-acp-trans-'));
  tmpDirs.push(dir);
  const script = join(dir, 'fake-acp.mjs');
  writeFileSync(script, source, 'utf8');
  process.env.DEVIN_CLI_PATH = process.execPath;
  process.env.DEVIN_CLI_ACP_ARGS_JSON = JSON.stringify([script]);
  process.env.DEVIN_TIMEOUT_MS = String(timeoutMs);
  return { dir, script };
}

const HANDSHAKE = `
import readline from 'node:readline';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
function update(u) { send({ jsonrpc: '2.0', method: 'session/update', params: { sessionId: 'session-1', update: u } }); }
`;

// A fake that completes the handshake then returns a chosen RPC error on
// session/prompt. Lets us drive each upstream fault shape end-to-end.
function promptErrorFake(message) {
  return `${HANDSHAKE}
rl.on('line', line => {
  const msg = JSON.parse(line);
  if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
  if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
  if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; }
  if (msg.method === 'session/prompt') { send({ jsonrpc: '2.0', id: msg.id, error: { code: -32603, message: ${JSON.stringify(message)} } }); return; }
});
`;
}

describe('classifyAcpError — transient-first ordering (unit)', () => {
  it('classifies high-demand/capacity as a retryable 429 capacity_error', () => {
    const c = classifyAcpError("We're currently facing high demand for this model. Please try again later.");
    assert.equal(c.code, 'CAPACITY');
    assert.equal(c.type, 'capacity_error');
    assert.equal(c.status, 429);
    assert.equal(c.retryable, true);
  });

  it('classifies "model temporarily unavailable" / "overloaded" as capacity (widened pattern)', () => {
    assert.equal(classifyAcpError('The model is temporarily unavailable').code, 'CAPACITY');
    assert.equal(classifyAcpError('server is busy, overloaded').code, 'CAPACITY');
  });

  it('classifies "internal error occurred (trace ID ...)" as NON-retryable upstream_internal, not a dead token', () => {
    const c = classifyAcpError('an internal error occurred (trace ID: abc-123)');
    assert.equal(c.code, 'UPSTREAM_INTERNAL');
    assert.equal(c.type, 'upstream_internal');
    assert.equal(c.status, 502);
    assert.equal(c.retryable, false);
  });

  it('does NOT let an internal-error blip read as unauthorized (transient-first wins over a 401 shell)', () => {
    const c = classifyAcpError('unauthorized: an internal error occurred');
    assert.equal(c.code, 'UPSTREAM_INTERNAL');
  });

  it('does NOT let a capacity blip read as unauthorized', () => {
    const c = classifyAcpError('unauthorized — we are currently facing high demand');
    assert.equal(c.code, 'CAPACITY');
  });

  it('classifies a genuine auth failure as 401 unauthorized', () => {
    const c = classifyAcpError('invalid api key');
    assert.equal(c.code, 'UNAUTHORIZED');
    assert.equal(c.type, 'unauthorized');
    assert.equal(c.status, 401);
    assert.equal(c.retryable, false);
  });

  it('classifies explicit rate-limit as 429 rate_limited', () => {
    const c = classifyAcpError('rate limit exceeded: too many requests');
    assert.equal(c.code, 'RATE_LIMITED');
    assert.equal(c.status, 429);
  });

  it('falls back to 502 backend_error for an unrecognised fault', () => {
    const c = classifyAcpError('something weird went sideways');
    assert.equal(c.type, 'backend_error');
    assert.equal(c.status, 502);
    assert.equal(c.retryable, false);
  });
});

describe('runDevinAcpProcess — transient-first error surfacing (fake child)', () => {
  it('maps a high-demand prompt error to 429 capacity_error (retryable, not a dead token)', async () => {
    installFakeAcp(promptErrorFake("We're currently facing high demand for this model. Please try again later."));
    await assert.rejects(
      () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }),
      (err) => {
        assert.equal(err.status, 429);
        assert.equal(err.type, 'capacity_error');
        assert.equal(err.code, 'CAPACITY');
        assert.equal(err.retryable, true);
        return true;
      },
    );
  });

  it('maps an "internal error occurred" prompt error to 502 upstream_internal and preserves the message', async () => {
    installFakeAcp(promptErrorFake('an internal error occurred (trace ID: zz-9)'));
    await assert.rejects(
      () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }),
      (err) => {
        assert.equal(err.status, 502);
        assert.equal(err.type, 'upstream_internal');
        assert.equal(err.code, 'UPSTREAM_INTERNAL');
        // message must keep "internal error" so reportRunFailure (special-agent)
        // routes it to reportInternalError (sticky quarantine), not generic.
        assert.match(err.message, /internal error/i);
        assert.equal(err.retryable, false);
        return true;
      },
    );
  });

  it('maps a genuine auth failure on session/prompt to 401 unauthorized', async () => {
    installFakeAcp(promptErrorFake('permission_denied: invalid token'));
    await assert.rejects(
      () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }),
      (err) => {
        assert.equal(err.status, 401);
        assert.equal(err.type, 'unauthorized');
        return true;
      },
    );
  });
});

describe('runDevinAcpProcess — graceful session/cancel on abort (P1)', () => {
  it('sends a session/cancel notification for the live session before SIGTERM', async () => {
    // The fake records every line it receives to a file. After we abort an
    // in-flight prompt, that file must contain a session/cancel notification
    // (no id) carrying the active sessionId — proving graceful cancel fired
    // before the process was killed.
    const evidence = join(mkdtempSync(join(tmpdir(), 'windsurfapi-acp-evi-')), 'rx.log');
    tmpDirs.push(join(evidence, '..'));
    installFakeAcp(`
import readline from 'node:readline';
import { appendFileSync } from 'node:fs';
const rl = readline.createInterface({ input: process.stdin });
function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
rl.on('line', line => {
  appendFileSync(${JSON.stringify(evidence)}, line + '\\n');
  const msg = JSON.parse(line);
  if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
  if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
  if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-XYZ' } }); return; }
  // session/prompt: never answered — we want the abort path. A session/cancel
  // notification arriving here is logged above before we exit.
  if (msg.method === 'session/cancel') { setTimeout(() => process.exit(0), 50); return; }
});
`);
    const ac = new AbortController();
    // Event-driven abort. A fixed wall-clock timer here races the real node
    // subprocess cold-start + handshake: under concurrent load (the full suite
    // spawns many child processes) that startup can exceed the window, so the
    // abort fires before session/new resolves — the runner hasn't armed
    // `activeSessionId`, takes the immediate-SIGTERM branch, and no
    // session/cancel is ever sent (child dies before recording anything).
    // Instead, wait until the fake has recorded the in-flight session/prompt
    // frame — proof the handshake completed and the runner called setSessionId —
    // THEN abort. Deterministic regardless of load. A generous deadline is a
    // last-resort so a genuinely stuck handshake can't hang the test.
    const aborter = (async () => {
      const deadline = Date.now() + 10_000;
      while (Date.now() < deadline) {
        if (existsSync(evidence)) {
          try {
            const seen = readFileSync(evidence, 'utf8')
              .trim().split(/\r?\n/).filter(Boolean).map(l => JSON.parse(l));
            if (seen.some(m => m.method === 'session/prompt')) break;
          } catch { /* mid-append: retry on next tick */ }
        }
        await new Promise(r => setTimeout(r, 10));
      }
      ac.abort();
    })();
    await assert.rejects(
      () => runDevinAcpProcess('cancel me', { modelKey: 'swe-1.6', apiKey: 'k', signal: ac.signal }),
      (err) => {
        assert.equal(err.status, 499);
        assert.equal(err.type, 'request_aborted');
        return true;
      },
    );
    await aborter;
    // Give the fake a moment to flush its append before we read.
    await new Promise(r => setTimeout(r, 200));
    assert.ok(existsSync(evidence), 'fake recorded received frames');
    const lines = readFileSync(evidence, 'utf8').trim().split(/\r?\n/).filter(Boolean).map(l => JSON.parse(l));
    const cancel = lines.find(m => m.method === 'session/cancel');
    assert.ok(cancel, 'a session/cancel notification was sent on abort');
    assert.equal(cancel.id, undefined, 'session/cancel is a notification (no id)');
    assert.equal(cancel.params?.sessionId, 'session-XYZ', 'cancel targets the live session id');
  });

  it('still aborts with 499 even if session/cancel cannot be delivered (best-effort)', async () => {
    // Prompt is never answered; abort fires. Even if the child ignores
    // session/cancel, the runner must still reject 499 and not hang.
    installFakeAcp(`${HANDSHAKE}
rl.on('line', line => {
  const msg = JSON.parse(line);
  if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; }
  if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; }
  if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; }
  // ignore session/prompt and session/cancel
});
`);
    const ac = new AbortController();
    setTimeout(() => ac.abort(), 200);
    const started = Date.now();
    await assert.rejects(
      () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k', signal: ac.signal }),
      (err) => {
        assert.equal(err.status, 499);
        assert.equal(err.type, 'request_aborted');
        return true;
      },
    );
    assert.ok(Date.now() - started < 4000, 'aborted promptly, did not hang');
  });
});